{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "308c07ce",
   "metadata": {},
   "source": [
    "# 数据读取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4ebbaeb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "63b8e05b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['val',\n",
       " 'train',\n",
       " 'label and phone.xls',\n",
       " 'parameter_get.py',\n",
       " 'some_package.py',\n",
       " 'preds.npy',\n",
       " '.ipynb_checkpoints',\n",
       " 'submmit.npy',\n",
       " 'eda0.ipynb',\n",
       " 'baseline.ipynb',\n",
       " 'resnet_best_model.point',\n",
       " 'baseline.py',\n",
       " 'catboost_info',\n",
       " '.DS_Store',\n",
       " 'resnet.py',\n",
       " 'dnn.py',\n",
       " 'model.point',\n",
       " 'log_dnn_layer_1_512_layer_2_256_layer_3_128.csv',\n",
       " 'dnn_no_fft.py',\n",
       " '__pycache__',\n",
       " 'test']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看当前目录下的文件内容\n",
    "os.listdir()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "09cd985e",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_path = './train'\n",
    "val_path = './val'\n",
    "test_path = './test'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "19aca406",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['10type_sort_train_data_8192.npy', '10type_sort_train_label_8192.npy']\n",
      "['10type_sort_test_data_8192.npy']\n",
      "['10type_sort_eval_data_8192.npy', '10type_sort_eval_label_8192.npy']\n"
     ]
    }
   ],
   "source": [
    "# 查看目录里面的文件\n",
    "print(os.listdir(train_path))\n",
    "print(os.listdir(test_path))\n",
    "print(os.listdir(val_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0685822c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 读取训练集，测试集和验证集\n",
    "train = np.load(train_path + '/' + '10type_sort_train_data_8192.npy')\n",
    "test = np.load(test_path + '/' + '10type_sort_test_data_8192.npy')\n",
    "val = np.load(val_path + '/' + '10type_sort_eval_data_8192.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7aac74ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 读取训练集和验证集的标签，测试集是没有标签的，需要你使用模型进行分类，并将结果进行提交\n",
    "train_label = np.load(train_path + '/' + '10type_sort_train_label_8192.npy')\n",
    "val_label = np.load(val_path + '/' + '10type_sort_eval_label_8192.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "69dc12cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({5: 2118,\n",
       "         4: 1558,\n",
       "         8: 1714,\n",
       "         3: 5745,\n",
       "         0: 1404,\n",
       "         6: 3398,\n",
       "         7: 1248,\n",
       "         9: 1386,\n",
       "         1: 3365,\n",
       "         2: 1467})"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter(val_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6c4181f2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(70209, 8192)\n",
      "(23403, 8192)\n",
      "(23403, 8192)\n"
     ]
    }
   ],
   "source": [
    "# 查看数据的shape，可以发现序列长度都是8192\n",
    "print(train.shape)\n",
    "print(test.shape)\n",
    "print(val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "26f24952",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(70209,)\n",
      "(23403,)\n"
     ]
    }
   ],
   "source": [
    "# 查看训练集和验证集的标签数量是不是对应\n",
    "print(train_label.shape)\n",
    "print(val_label.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e3a68c53",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 0 ... 9 9 9]\n"
     ]
    }
   ],
   "source": [
    "# 查看标签结果，注意分类标签应该从0开始\n",
    "print(train_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ead1fce",
   "metadata": {},
   "source": [
    "# 数据处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "33a78586",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据归一化\n",
    "from sklearn.preprocessing import MinMaxScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "91488953",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "226c59e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f9057741f40>]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO2deZwU1bXHf2dm2Bn2YR1gWEYQVEBGQBRFRUEwAX1R0RiXmBCJC+pLIhhjNIqaRMWYpya44Y4aNaAsAm64gDgIyL4jDCD7MoDMMDPn/dHVUF1dXXWru5buqfP9fPrT3be227er7rn3bJeYGYIgCIIQJSvoCgiCIAjphQgGQRAEIQYRDIIgCEIMIhgEQRCEGEQwCIIgCDHkBF2BVGnWrBkXFBQEXQ1BEISMYuHChbuZOc9sW8YLhoKCAhQXFwddDUEQhIyCiL5PtE1USYIgCEIMIhgEQRCEGEQwCIIgCDG4IhiI6AUi2klEy3Rl9xHRViJarL2G6raNI6J1RLSaiAbrynsT0VJt25NERG7UTxAEQVDHrRnDJABDTMonMHNP7TUdAIioG4CRALprxzxNRNna/s8AGAWgUHuZnVMQBEHwEFcEAzPPBbBXcffhACYzcxkzbwSwDkAfImoFoAEzz+NIZr+XAYxwo36CIAiCOl7bGG4hou80VVNjrawNgC26fUq0sjbaZ2N5HEQ0ioiKiah4165dXtRbEAQhtHgpGJ4B0AlATwDbATymlZvZDdiiPL6QeSIzFzFzUV6eaXyGoHGorAJTFm8NuhqCIGQQngW4MfOO6GciehbAB9rXEgBtdbvmA9imleeblAspMO7dpXh/yTZ0yquPU9o0DLo6giBkAJ7NGDSbQZRLAUQ9lqYCGElEtYioAyJG5gXMvB1AKRH107yRrgUwxav6hYUfDvwIADhSXhlwTQTBmn2HyzFv/Z6gqyHApRkDEb0BYCCAZkRUAuDPAAYSUU9E1EGbAPwGAJh5ORG9BWAFgAoANzNztNcajYiHUx0AM7SXIAgh4BcvfI1lWw9i3fiLkZMtIVZB4opgYOarTIqft9h/PIDxJuXFAE5xo05CMBz48Rh63D8Ls+44Bye1yA26OkIGsWzrQQDAnJU7MeSUlgHXJtyIWA4Jfq3t/dC0lQCAiybM9eV6QfH615vxk39+EXQ1qiXzN/ijTlq7oxQTZq/x7dnIJDI+u6pgDZk6e3nHscoqX68XFHe/tzToKlRb/Oqor5w4H3sPl+PGAR3QoHYNX66ZKciMoZrD5h6/1Y4te4/gUFmFL9cKg/DbuPsw/jZzVSCjab+ueKyi+v+PySKCISRU57RTz32+AQP+9gkufPwzX673r0/X+3IdPQs27kWf8XNQevSYL9cbPGEunv50PV79erMv19PjtyyaL55QcYhgCAm+jfwCkD9vfhMJpN9+4Kgv19u054gv19Hz6KzV2FlahuXbDvpyvXJtVvTOwhKbPd2nymfJMOqVhb5eLxMQwVDN8dvGEAaCUK8cOOLPTMHIpj2Hfb9mOJSf6Y0IhmrOgk2quQ1dQp5qT1i9oxSA/2qW/QEJJCFYRDCEhAUbfRYQ1RmfJ2F7DpX5e8GQUOqTs0ImIoIhJDw2e40/FxLNleus3Xko6Cr4ioQVBI8IBp+59Y1FmLF0e9DVqDYws+8dp9htvEYkQ9CIYPCZ95dsw+jXvg26GtWGuWt3B10Fz9GLocnf+O8+KoQPEQxCRvNjAFljgwwanLJYMtEL3iOCISDKKrzv0HYe9MevX4+oWdynOgcnmvHGgi32OwmeIoIhIMp8CMffsDsIH/Tqrx8W4ec+fgyU0oGjxypx39TlvqVvSZZQJ9Hbf6Qca3YcQp8OTXy/dnX1vNi+3/9Zit/8V5ZKdZ27/vNd0FXwhdPun4XyiirUrpGNsRd3Dbo6CQn1jOEXzy/AFf+eh8oqf3rpbft/PPHFh0sGMa6d51PK5ETsKvXe59+v+yVMzFz+Q9BV8IVyTVPgd9oPp4RaMCzfdsBy+/4j5SgYOw2frdnlyvX0axSEQeXiV159PZJbPzM5eizYTKebfch/tWTL/uOfy9M8s2uoBUO0C5m71rzjjyYs+/dn7mTT9FuvGLTRctV2fxK+6amOYiFo23MYhO3bC703eH+VQVlcwy0YtPv9l5O+Ceza3l4j2Af6tQBSNgvu89JXm4KuguAzoRYMUez6z0yS9OmEHxHJB31anyBIgvaBWr/Lf+82v/FjDBX0zM8JIhiqMUGrkvzgDwZvlhBoPQQP8MPml0lPowgGRX5weRGYY1XpbXwS0oeKgL2gwuAo4UcTPzxj1fHPQat57RDBoMhWvaupCzzjw/KQIZgwxOH3b/bD82rkxPmeX0MQ9IhgCAg//O3DiN8DsRU+LbUpeItb982DH6zAGwsy3+nCFcFARC8Q0U4iWqYra0JEs4lorfbeWLdtHBGtI6LVRDRYV96biJZq256kaqwk96P/MjaeXwvJh4l0D1RygzDE87mlLnvui40Y9+5SheulN27NGCYBGGIoGwvgI2YuBPCR9h1E1A3ASADdtWOeJqJs7ZhnAIwCUKi9jOf0leiC6EAkXbarWNwZBWOn4epn3VcfbHfZTmKkKgw9iIEJfi2A5DKbdh9Gj/tnxUbjJ2DLXu+DvwInfLeuJa4IBmaeC8C4duRwAC9pn18CMEJXPpmZy5h5I4B1APoQUSsADZh5HkcsMy/rjgmEG148Ed8wyWVf7hKbB9LKRXbR5n1JpWVIdXBbXlGFKYu3JjSclR51P4Bv96EyfLJqp/L+fhtKD7uQ9puZMWXxVl+jYSd9tQkHfjyG/o98nNJ53vxmM/67yP3cUX5HBrtx1xyrVK9zuk80vbQxtGDm7QCgvTfXytsA0IcZlmhlbbTPxvI4iGgUERUTUfGuXe6kq5izYocr51FFHx6v5+1i6wjMxVv249Knv8I/PlrrRbUseXz2GoyZvBgfO+ioU+Wa577GDZO+wdFjyXfAK7cfRMHYab6kPUiGj1buxJjJizFhjvrso/ToMRxOIZK+wgWvuJ0Hj+Kud5bi9jcXp3wuI+noCbX/SDnetFgoycnsMR1/n54gjM9mdgO2KI8vZJ7IzEXMXJSXl+dKpZINxpq5bDu+3+NOANCWvUfwe5ssk09/sg6AWroJo4UmmZuRmXHf1OVYt7MUO7T1HQ78mMBW4YFFaNUPpVo91PY3S4n9dnFkvDFrRfokalu3sxSLNu8DAHy5PrIK3Q4H62ecet8snHb/rKSvb5U6fMMutWfhkn9+kfT1gyba9lFU3EfvfGsJ7npnKS58/DPT7U878DTcdzg5e9+u0jLsP1Ke1LFO8FIw7NDUQ9Deo8PMEgBtdfvlA9imleeblKc1N736bUxyvFQoV5iKztJmNrMsZjhjJi9C/4c/cqVOm/cewaSvNuEGXdqQIKbBqkLNjZHYx6t2oGDsNEcdtVMGPT4Xlz79FQDgxS83JXUOvTrx3W9L0O3emUrqjG37f8Qr879PuP27EuvkklF2ZrBn3aLNsTN2lXt696HI7zUbRDq1w0xLct33M8bPQc+/zE7qWCd4KRimArhO+3wdgCm68pFEVIuIOiBiZF6gqZtKiaif5o10re6YtMaPRXecMGXxNmw7cBRlLmasJN0Yc3YCoeTUh2xn6VHlQJ8LH3dH+Krw2vyIumCpYgeZiB0HjzrzBEtBpv3lgxU4Ul6J1doMy4p3vy2x3O6FL+DRY5WuOCfMW78HixOoYVOlqorx15mrkhoQqAzqkoGZA1m+1i131TcAzAPQhYhKiOhGAI8AuJCI1gK4UPsOZl4O4C0AKwDMBHAzM0d/+WgAzyFikF4PYIYb9VMh3XV+iaisYuw5ZD5ye2jGypTPv+9IpGOrqKw63kIzl/+AP09ZlvggBZaWHECf8R/hLRubSpRUAgxf+HIjAODBaSuV9PIfWdhQnDykfR/6CEOe+Fx5fzeIqnfKKiqVVQ6J7h+3OHqsEl3/NBMPu3A/XvXsfIx46ktUVTGGPDEXBWOnme63+1AZfv/2Eke2qenLtuOZT9fjf99aYrrdSl4at328Kjmb5e5DsaqiV+Z/j5Pvnel6gK0dbnklXcXMrZi5BjPnM/PzzLyHmS9g5kLtfa9u//HM3ImZuzDzDF15MTOfom27hdMsbryqijFz2Q+2I591O0uP2x0Kxk7D47NWK50/GVvFAx+sQO8H5xxPJqdvMuNqasm05vhpKwAA2wyuri/NS6yKUGHtzsjIdv4GozNbahh/o7Ejf+3r1Ort1GgbfaB3lh61X9dDcaRupi4yHnrDi98oqxysBCGQuuow+h+8vdB6pmLH3sMnOswZy344bn+KUlnFx5cIfWj6Sry9sCShm7nZT7rl9UWR+iYSJg6mUv/4aJ3Sfmt3lGLh9xF7x5An5qLowVhV0XRN5eSWHVMViXzWUFnH99Wvv8dNry5EDxuj36DH5+Lcv396/PuTH6vdJI/ocqmo8qG28tXfZkaOff6Ljce37TmsNmJkZjw8YyWWbY2oTmYs3X58ZF1eeeIRsmsh1cfm4NFjuFMblXkt+40zQbv/2a4+qjGXRpXOFf+ah+teWBC33z79f6TYFOsVjMOOMgIbrqvyG4MYs+ldWM1mfje+9A263DMzpuzbzc7VTkbDtArGNjukqEK8cMJc/M8zEVuTUdABwPcBedKJYHBA1MBU6uOCOweOHMPIifNsA5Fe1fTi73+X2KiV6Fn+ywcr8O/PNuCSf36B5dsOYPRr36L7nz+0rVuyS1wWb4qdJewsdc/Im6p+fM0O605X9fSDnzhhEzlcVoFNCR7wW99YpFo1S6Iqv1SYv2FP0ioQFfY7qCMz8NznG3DTKwuPlx0uN3/uoullPl19Yka2dV/keTGmp/j1y8W4883Fcf9jqmLOeD63UpVHg1Kjgza/EMHgALczdBhHXWaj2SlLtmL+hr1KSffsRnFHEjxYeq+Yw2Wx02h9jTYaprPfGkZWqu0TFWIA8OmaXegz/iOlDmm3gi5cdSB739TlmLI4PjBLr6YpLYvvyLKSuAc27k7cSSSTMyuZwfrC7/ehYOw0fLt5X8wsEDgxqxo5cT5+Oak47tigEtM8OG1lzFrQFzx2wk00qn4BzDMff73RXEU5e8UOvGsTkJeoeTcquvGqcsBGUOqXHp4w29+4JREMKZBqAjWVB/yz1dZ6af0zm1A3qnH/+ytM6qDeyxhd/KLT+dKjx1Awdhpe1KmxrNAHyEVHkYu3xI+IPlkdq/t+3cUV4SZ9tQljJkcCs/SxGnrueDOi7lpacgAFY6dhyuKtSsIpWew6rGSIGl+j9o25a3bZ2sjiRtMmuyerSTL+pwnrYCOM3tQ5LdgZtSfOjR9UWVU/0W87aBLZ//D0lUmP5jfsthY0+kGa384xIhg0VHLGGO9Vs5TLRjVJqkQNgwyO68QP/HjMkatsyb54dcanBoNoolmFGddrKUOi/uxmvvFrdpSiYOw0W8PrOyaGSeMiPCowYo2QZhgFQDRWY9Djc+NsA8cqq44brMdMXoyX521yXifDM60XLlb1TIRehVd69FjcOfSd7x/fi/ces+t0k5khmA0wjh6rjCv3Ihut0Z7yjeEZfGh6vO3uCUOU+fMmg5qCsdNwxb/mJbzuIzNW4d9zIyrYvUkEnTnxNPLbpBNawfDF2t0x360CfqIYR8xGyiuq8LMEN5KZJ4mT/3rp1oNx+Zp63D8rzsCcKNUGYK6H/sYw5X5VoR3i6qb5+5t1KNGHdOYy66hjs4fE+DCouB5u2/8jOt09HV3umYnKKjY16I3S6a2N/O/bsa6Kj81KPUneG7o0Clv2HomJAzHaHlRyBD047cTM74UvNuH3b8cKUH2Or0VbjBG+5naSebrO1ajSNBssTDV4+7z/XSQDQIV2nx8pr0DXP81Eh3HTMUP33//9QzUPvVQ6QrNBnnGWZJfXK2pPXGAx0PvXZydmIpdpwYpOiHpBJUL/PPlt6g+tYFhS4txbwXiT6P+4q5+dbzkCMPMk0etJjecDYjuJJVv24wMLw7Iq3+85fNyTyeyaRlSCiaK5cnYctFKzRG7tnQrBQxWVVdh7uDxObaOScmCMzpj7yIyVeOub+DiJDQ4Mgxt3H3IcvPQ7g3DRC8VX5n+PLIs2V7mW3sWXyFroRl2W52ozNmaO+9OZrY2bZp49xvxIK7YdxLl///S4Z90hXcd793uxaahV1JdLDfVxkhfKzClikcOgOP1Artu9M+PSwPiRd2vqYp3wZX/T5odWMLjBc5+fmH5+tX4Pbngx3h0xilmgk10yujGTnXmsHFLIbnru3z/Fb3QjZuPo0GkUd8K8SRrLDaoDlXQLf/lgBU5/IPWw/w+Xxxu0zWwEVv1UFTvPGvsfC3/9iXM3WBrpH5ruPAjMSphE7U5RAf/kx+vwpCEB4+frdscd55Rou87faO8m+/la++sdNNxXdveZnqc+iXcPd7puht7J4Eh5ZdxMWjWeZe/h8liX5ASYOSHotRgMxiEfvSFDKxjMnk1mxtQl21BeUXV8SmyFcYZgVAvYuXPuPHg0xvPAyAzDSNDu5n7+SzXjrxUqD62eR21UA04MxtHZxMspBs/pMf7P/567IW6fOSsTe0Sp9Cebdh9GwdhpcbrtKHsNHYPVJC3aXofLKmzVb24xzTATddIJR3HS70Ztc7tKyzBruflvvPGlE95Rizbvwz4HOnwz12BmZ44WRu8zowpM9UynPzAbvRQGOat+iB1AGfuFY5WMbB/dw0IrGMz4ZPVO3PbGIjw+ew3mubCW7wffWecAfHfRVgx78gtlTw07G4eThzN6Ixqn7E5xkoPeDieqss53T8dfZ1oHBO4+VIbiTfbBSlaLMJXsO3LcJx4wb+OBj34KwD4HUZRnPrNXid31zne46dWFWLPDPveRG+gXcTKqfpzw/W57FUtUJXjG+DkY9crCOJWqWd2GPameydU0TTMzXnew5KZdH+zUBqKiQtVjF5j3xoLN+GzNLs8CDUUw6Ii6Tv5L4cFVQTVrZtRYaGYodYKTJGULNKOzbZoGG1QHMdH7186l1o6oZ1VFFZvGduhb4Eh5ZdLp1KOs+qEUK3QpzpNxczSiYuPYogmjZDyzkuGFFGebX6yL3EfJBH8+bKM+U5nBmMWkGCnZp+4F5HZSvFdtZs7Ge2eRjbAc9+5SXPfCgqSztNoRWsFgFkym7+SsDalqOMkC6UYGxdUOR5dGt00jqSyOY2SyZgS2Wg9XpV/1YpGgJQ6yqH5qMbtzqq6wIupd5lUm0WTYfagsYScd97w40HoU23SCKujtfRUuZHG91cZjSOVu1T8/RruOHWYxLWaBkmbBfW4QWsFgh93I1ujumipb96fu5eDUPjDIJpW1ysh2/U5nof9uGND+MSfxQ+Z0jQCnqjC7+jvKUZRhFD04B30fmqO075Ey9wYVbmjWnYqKFTYLYf1wwP4+M4tzSoTKzPvKie6vA5+I0AoGsz9CP4t4wCRKWM8DH1hvd4rTTt0PVDw59hx2LxL4NcUYCqslMJ3mb1JJSKfH6vRvFm/BkQBy5/vJUYU1PqYs3qocr6CCyj+qYhNwYz2IKCrPRqK0HGakVx5pICfoCgSF3fKFdjpGp2obO/5PMQOrn6h0sk7uZ7vo8g0WOYW8wukD6TSdRDKMn6Y+6HAzAaFbvF1cgpxsfxMs2V2NmZPytkp4PtfOlJ6EdsYwZ6V/C9qr4MQdzw1UImw3uZwDXkXP6uaozgvsRopZLjxRz36ubgjWJyRMRBApsj+1yfHl5H92Q8T8yyZ+xClu36dBJSpMRGgFg9n/8OC01FeYSha/+0OV36ryIDmJIj6oELnplZdFIpwKZLs2UVnXIxO57OkvlfdV8ehx4g7uhuv43DW7XO18VVyOneBG2nQ3Ca1gMMPLzJmZiNsBNeUV9tLPbn0Ct3X4B390Zgw3BquFBb1fvV3iP5VEeU7cQacsto4HAqA05LZKReKUlTbGacDZTOc2l9blcIvQCoZ0m7qlI+63UerTomRWufOTLSYZbIPGaZoTO6zWlwDUVFf6RH9+oaJ2U0VFO5fJfUxoBYM7msvqjdstlJ7mA3crde+U5a6ezw2u+Hfi1NHJMMkmcDOIv9kqq3BQqHhwpYpX5qMQCwbBjmMu9+QLHLjv+cXsFenlhOAFKokLnfCRB0GGmYbKoMmvdCZeEFrBkMnTPL/4+4fuqm38zA6pyjuK+Y0EddLNJ98L0uUnetWPhVcwBF2BDGDLXvXcMkJ4sFunOtV8WJmAih3Faarv5OrhzXlDKxgEQRCS5bCCd9yX6zI3PYrngoGINhHRUiJaTETFWlkTIppNRGu198a6/ccR0ToiWk1Eg72unyAIghCLXzOG85i5JzMXad/HAviImQsBfKR9BxF1AzASQHcAQwA8TUTZPtVREARBQHCqpOEAXtI+vwRghK58MjOXMfNGAOsA9AmgfoIgCKHFD8HAAGYR0UIiGqWVtWDm7QCgvTfXytsA0K/eXqKVCYIgCD7hR3bVs5h5GxE1BzCbiKx8IE1X5YvbKSJgRgFAu3bt3KmlIAiCAMCHGQMzb9PedwJ4DxHV0A4iagUA2ns0YqYEQFvd4fkA4hKlMPNEZi5i5qK8vDwvqy8IgpC2sEcRFZ4KBiKqR0S50c8ALgKwDMBUANdpu10HYIr2eSqAkURUi4g6ACgEsMCLuqVLgIogCEK64bUqqQWA97RUxTkAXmfmmUT0DYC3iOhGAJsBXA4AzLyciN4CsAJABYCbmbn6R8sIgiCkEZ4KBmbeAKCHSfkeABckOGY8gPFe1guQyGdBEIRESOSzIAiCEIMIBkEQhAzFqxUDRTAIgiAIMYhgEARByFAy0l1VEARByDxEMAiCIAgxiGAQBEHIUGShHpeRyGdBEARzQisYBEEQMh2vBrihFQwS+SwIgmBOaAWDIAiCYE5oBYPYGARBEMwJrWAQBEHIdMQryWXExiAIgmBOaAWDIAiCYE5oBYPYGARBEMwJrWAQBEHIdCSJnsuIjUEQhExHjM+CIAiCL4RWMIiNQRAEwZzQCgZBEATBnNAKBrExCIIgmBNawSAIgiCYI4JBEARBiEEEgyAIQobCHvmrpp1gIKIhRLSaiNYR0dig6yMIgpCuLCk54Ml500owEFE2gKcAXAygG4CriKibF9cSd1VBEDKdZVtDIBgA9AGwjpk3MHM5gMkAhgdcJ0EQhFCRboKhDYAtuu8lWlkMRDSKiIqJqHjXrl1JXUjcVQVBEMxJN8Fg1l/HaX2YeSIzFzFzUV5eng/VEgRBSD/CkiupBEBb3fd8ANsCqosgCEJaQx6pPtJNMHwDoJCIOhBRTQAjAUz14kJifBYEQTAnJ+gK6GHmCiK6BcCHALIBvMDMywOuliAIQqhIK8EAAMw8HcB0r68jxmdBEDKdsNgYBEEQhIARwSAIgiDEEFrBIMZnQRAEc0IrGMTGIAiCYE5oBYMgCEKmwx7pPkQwCIIgCDGEVjCIjUEQBMGc0AoGQRCETIc8spaKYBAEQchQxMbgMuKVJAiCYE5oBYMgCIJgTmgFgxifBUEQzAmtYBAEQRDMCa1gEBuDIAiCOaEVDIIgCJmOpN12GbExCIIgmBNawSAIgpDphGXNZ98QG4MgCII54RUMIhkEQchwxMbgMl41qCAIQqYTXsEQdAUEQRDSlNAKBkEQBMGc0AoGMTEIgiCYE1rBIKokQRAEczwTDER0HxFtJaLF2muobts4IlpHRKuJaLCuvDcRLdW2PUkkvkOCIAh+4/WMYQIz99Re0wGAiLoBGAmgO4AhAJ4momxt/2cAjAJQqL2GeFw/QRCEjMUrzUcQqqThACYzcxkzbwSwDkAfImoFoAEzz2NmBvAygBEB1E8QBCHUeC0YbiGi74joBSJqrJW1AbBFt0+JVtZG+2wsj4OIRhFRMREV79q1y4t6C4IgpD1e6dpTEgxENIeIlpm8hiOiFuoEoCeA7QAeix5mciq2KI8vZJ7IzEXMXJSXl5dc3ZM6ShAEofqTk8rBzDxIZT8iehbAB9rXEgBtdZvzAWzTyvNNyj1BvJIEQch0Ms7GoNkMolwKYJn2eSqAkURUi4g6IGJkXsDM2wGUElE/zRvpWgBTvKqfpMQQBCHT8aofS2nGYMPfiKgnIkJtE4DfAAAzLyeitwCsAFAB4GZmrtSOGQ1gEoA6AGZoLyEgOjSrh427DwddDUFIO05qUR9rdhwKuhqeJQP1TDAw8y8sto0HMN6kvBjAKV7VSY9ESAiCkCzdWzdMC8HgFaGNfBbsYdG3CYIp3Vs3CLoKniKCIU1oVLdG0FWIIysNp1Wdm9cPugqCDTWy0+++qa7Iegwuk26D4Xo1vTT3JInLz/cL1xelfI50n8UMKGwWdBUCh0LgDO52tp4GtdPr+Q+tYDCjSb2aQVehWnN+1xZBVyGOX/Rr7+r5urWq3ioGJRT6zD8M6eJ9PQw0rOPerFxlgFIrR717rZtmA0MRDDpOb9dIed/cWu7+keeclH4jzXScxbg9UrvhrAJXzxcGHhyRun9I73aN7XdymQlX9vD1eo9d4f31urT0RrUaWsGQav9Sq0a2/U4OaNWwjqvnc4OLuqXfCN9tOua5/GDZ3Fc5WfY3XrbCPk74ed92rp7vou7W94VK7Ws6GE1nKk5mKKwQqpaXWyuuzKuZRvX/dxJgNhOs0pW9/uu+/lUG/ts8GisYu7Nc7qDsaNNITTie1yW5NCh+oGKw79oy14eanGBEL9OUY4647fzOxz/b2RBUBl01sv3teq4oyrffKc0ZO6Srb9cKrWAwo1InGU73earrcx+M/MZ1bfcpTEMPoDo1svHstakbsYOkVcPaKW13SicXZkV3XtQFrRXrdbLLdhYVFa/doOIvw30Jj0oaFYN9jom3V1om0ctkzEY1VQ6G7SoqASeGSL9H5ypc1L2l79e895Jults75tVDjs+jTbexu8tm3n7O8c93D019lOiWU8Wrv+qLMRcUoll96/NNur6P7bnM1CKJ+MfIXsr7JiI7ixzNys/q3DTla/qBVx7lmf2EpYDZTaKqygCAVo3sR09PXqV+Q5vplX/WO3b6e85J7qlQsggYe3HqnU6XFu6pRcd06hIAABigSURBVIiAszpbG+GbO+hQgqBnW+vRLZG92rC+zrEhnWJJOubVxx0XnmTrANBQQU3ZosGJ56eOjb2ubRP72a3bjOiZuvrNCX06NIn5flILtVmeV67BIRYM8U9nfQeeRk3rudtB/aRH67gy41/eoal7D0h2FuGU1g1dO58qF5+SeBbSo20jWyPc7wan5uaY39h9I/9lmg7/om4tMNhmlqX6IP/yrA4p1ytZLnTB6aCvoaOzwo/JcvqIV3OSHfDIjMFlzNRGd1x4EoraN8Y7o8+0PV5FleQEJ7OVRDixCeRkZSl5QjghVyFIp3f7xLabXw/oaHt8rZzI6PKeYSfjqatPt93/0ctjXQZTHYGbdXiPX9kTU285C0+M7Gl/ApvLd2xWD8AJfbJdfd30zY9Sr6a9x53d7PWpnyf+b1o0iO0ET833f4DiN00dqvP0M6ogCLFgiC+rVysH/xndH73b2492HnDoy/3wZafGld3l0MvArht3ohrq1Lw+GtVJXfd844ATI9u7h55su/8NFiPhk1vlomMzNeH2qwEdMey0Vrb7uR1R+j+nm3u3nJbfSMl10E4s/Umzsdx6fmdc378AV5u4ml6uUzEGpWm6oX+B5fZm9ROPgE9tE6tu80NdRuTMxuDGkKmeAw2EsQnuGtLV8nm+sqit6XFuEVrBUFdhVGSFE+MZYD6y69rKWj/v9E93YpS9e2hXV0ZqVxS1td8JQHtNDWblo18rJ9tz/3azNh1o4f56VR93YwCyiCw7nWhdcmvXwH0/7Y7aJvp3vc79wpOdq32cROR6gYqwNhOIVtx0rvVsM9VUKkYVaN8O9sbpVDwba9fIshRk7ZtF7gGxMbhMdxv9ejKSeNINZyQ+n0mZndeS8U83dqqv3hgba+Gkyrm1I4LKqGrxik/+d6Av17nU4LNvNJSa+c+fU5hYMHRoVtdS/eUUu/vKaWT3Nf3aH59lmGE2wzFe4tozY9OCqHShidSQT1uokKI0UFB/DTs1djZop2r9xZkFtud0OpjTYxSm+kGVlQfTa79KHA+lH5xeFvc/UVwb6wVF1AX5NI/UcKEVDMYMkDWTcIG8fVBhzPeBXZon3NfsMWrRoDau6tNOOTL1lvM6x3w/24WEbUbf9DEXFCbY0x6zLi3qXeGXO+65Nrpvs1pcccaJWU/HvHpx2yeP6pdqtWKu/6uzUzMs638DEXCj4Xx/VFDp6flFv/bKXjBRjKPZ9k3rYtMjwzBU16G/99v+psca3V3NZpHGkvdvPVu5bmYOBkSEHjYeY1ZYCeyo3cuM0/IbonluLdxx4Ulx2/Rt2N7EsSTb4pqDu7fErDvOUbLLJUNoBYMx4CWZtNd6l7Zo+oi/DO+u28N+7PXwZadi/KUR+4OZQW94zxPeSvV9yMDY0uXgqnd/exY+/8N5jo6JqhqMnZVKOmfjs2R0MjDTZ+u90abfNiD2fCBXo3RPapnreeSznXunsVMnAuro7CMqItx4DjNHgF4JVCm/ObdTzPe//8x+1mqMxTjbwq3Zymah0vbGmTgAU5VeFELipIC5tWtgwR8HoajAetZZt2aOY8+kk1rkejbgCq1gME4rVWbwRruE/piJWjRuXgKjm8rfd+HJsTMOIv+jj1VtBnqm3XY2Rg/shBYmQqV+rRwlP3T9g/7x7wZi9h3n4HcXxT5sKvpUY4dVURlb0KFZPYw8I/FvNHYARv/yvAb2D69RoOl16vVq5qScC0l/35m1iZWKckBhM9xmmBUyA5OuT6wGTQWzkbBR0KoORvRqsXt/Yh0Iqadn20aOXGL7dWwScz+OHtgJ4ywCDYmA5rmpD6isnDf8djIIrWAwYuxQzKZxxlWbrIxDbhlRRw/sjJ/1zre0X0RRuXlaN6wdk/fG+BOcdFpX9Yl0sN1bN3TsYWVFs/q1UNgiNy7B3QMjuic4wpxnfn46zjCM1IhwfIZm91NHD+x0XP2w6ZFh2PjwUCUBP+uOc2O+N9MNQq7u2w5NLTx2VLCzQ7TTdcbGXXu1bYReJiqVxvVqYtMjw7DpkWFKNoYzO6lFBs82tIWe937b31HG0/O7mqtqjXm/+nWMFeYvXn+GY9tNa82m0aphbdw1pCsa1E6sUWjXJF79aMRs1mm0DVk9e7een7yKNxlEMCTAysPnpV8mDvkv1EaLDwzvjg6KrpeJIIrcLI9e3sPSfhElO4twvY0b4VfjLsCdF7mTCz9Z/abRQKyKSn4nPRef2grNXfQHTzblt94AOdRgVE1m8mAXRKendo3IfbziL4Px5qh+GDMoXtedzM9SdcW0GiD1atcYl/ZKnNzumZ+fjn86yB4Q5cER8a7hUZrapPMATvzPH95+DqYZVIt6Ft4zCD/rnY+7h3aNG4AYMctXZeV5RQQ0rhtb1yEWwaFuI4JBQ+XhOLtzxAbQoWm9hMd0bp6L7+67CFee0Q7ZWYSvxp6Pn/RojfMSjHb0VBqCK6x8/s3o16EpLrHw7W/bxMQoZ7JfzZwsJX1+h2b2IyUz/jjsxJQ5qioza0tjmcoszN7rx/YUrnDzeSf06K0a1sEfhnQx1Ytf1O3Ew26mdjFDv7yp3e+JzuTq1sxB345NXU/p7RkUEexmGQG0zSc+GxrBeJ/obXNPjuyF8ZeeYmlTjLq2dmmZa5lnqmn9Wnj08h7Iyc5C+6bJPQuJqFMjOyYlTq2cLEs7h9uIYHDAred3xrxx58dM1c3QTztbN6qDf17VS+lP/akhP8tJhjxENbLi/66oh8dtFxQiK4tQVJA4OO9jE5fRqCjq0bYRFv3pQgDAd3++CEvvGwwAaJlgxO00klOPXk2nFxJ2FCm6jc4bdz4++/1A02159Wsd71Ts3FBVulBVQ/JvB3bGq7qZQ3Qkf15Xb1OI55qoQIyqIis1CRCZ/dphFak7644TSQFXPzjE9lyJ0C9mpRcGUbXWWZ2bxg1+Nj0yLEaN07R+Lfy8b3tTNXC6yMyGdWqgdaM6yMoi3KM9H36vCZ9+S3SlMVlZ5OmCOnZZMM08EKbfNgCb9x4xFQg/7dEaU5dsO/7dyrumXs1sNNaurxdi+hFm2yZ1sGXvjwCAOXeem7RqJbd2Dto1qYuxF3c9bpju38ne9Vb1elb/0bihJyMri/DBrWfHjdC98gk3Y/S5nTFhzpqYnFvJtGbU0PzU1afj41U7cUkP+2hwI0Z1m7EeKt5wVjEC+gGOlWtnTJ1Mzmcm5ADgMS0W57VfqbsV3z6oEPe/v8J0m9urBKrSRRtk3KMbLF3fvwAN6tRIGHHvFSIYNJyqbYATqycZg3H8pHmD2gn16F1a5gJLrI+PPgOJVAy/G3wS7ngzchL9SD8VN7mc7CzM1bmwfjX2fNM1CJIJSLJ6qId0b3lc6J3SJlYIzP39eWhi0D972T/cfF4nnNwqFxecbK9itCL6Pww7rZVSihAVXF/VDhHvLiexQp2bq2cFMJuNz7x9AA6XVSQ8Ppmgxab1amLP4XJllZ9TTmoRUUPrZ3A52VlJeQqmiggGjbMURqxG8nJr4b3f9kfXlum3ALxqp9q3Q1PceHaHhIbkS3vlY+fBMvTt2BTf7zmMMZMX44kre7qavK11gqhWOxWHGVZ9uVVHr1cPnt25Gb5YtxvtTbxNVBMrDCjMw1OfrE+42lxOdlbcehfPXNNb8ez2XN47H2t2lJpuK7QJZrvuzAI8PnuNa3UBgLd+Y5+YUoWOzephw+7DACLea7sPlZnul8wzefGprTDtu+0J76HpYwbgvqnLLSPNU8Xunl/94BBfVntMSTAQ0eUA7gNwMoA+zFys2zYOwI0AKgHcxswfauW9AUwCUAfAdABjmJmJqBaAlwH0BrAHwJXMvCmV+jn7LfFl799yNn7yf19YHpcoiMdrxlkk2Fr1wBAQAc99vtH2PNlZZHujRwOSerZthOE+56l3g74dmuDrjXuV93/lxj6Yv2FvnNujE/p1bIoNDw11NLNysvLZs9cWWaYQ/7tFqpPmubXRq10jLNq833wHMn5NE+U7gKevOR3Pzt2Igqb18P6tZ2Hl9oOunXvCFT3x50u6JfzPWjSorSy8nazF4gRVVVyqpGp8XgbgMgBz9YVE1A3ASADdAQwB8DQRRX/RMwBGASjUXlFr1I0A9jFzZwATAPw1xbqljN3Iygt+b7PewK8H2Ku8atfIRq2c7MAybwaJ2W9+UMuEe56Cy2/kHIQzOzVNWdfsZRqQC7u1cH0JzSjpfN90bdkAj13RA9mave/8rqmvHRGlZk5WSu7N+jU0BimqCHvkN3R1sSu3SEkwMPNKZl5tsmk4gMnMXMbMGwGsA9CHiFoBaMDM8zjiE/YygBG6Y17SPv8HwAXkoxXILvWvXQ4et7jZkA/JSLRz69vRPsDIjylnumE2ui3UdLdXWEQ8q+JkuVZVnryqF4ae6v8yqoB5Ongjbq/bUV3RR2PnmHgQmjHllrPxoc5rK13wysbQBsB83fcSreyY9tlYHj1mCwAwcwURHQDQFMBu48mJaBQisw60a+dOWmQ7EWSVm8VP+ndu5lhF4Rdm7rTpQjL2CjOMbplujF1+2qM1fprAX99rjC7RTqhXMxuHyyuTjmcJEq8HTV6nj/ca29oT0RwiWmbyGm51mEkZW5RbHRNfyDyRmYuYuSgvz52RvNnzna4j7nQUCgDQXzFNQjLo03hY4bcaJD3/CXuieb9SCXh7/dcR91Cr3FNO6xMUKgGdYcJ2xsDMg5I4bwkA/d2SD2CbVp5vUq4/poSIcgA0BKBuMUyRdDKwZSpGgWXMYZMM2VmEyipOmMZj2KmtMG3p9uPfvXIlrG5MuLIn3lywBT1MYjdyDekuEj0bPdo2wpdjz49L3Z4MLRvWxoZdh1M+T7I8ctlpgV07HfFqvjMVwEgiqkVEHRAxMi9g5u0ASomon2Y/uBbAFN0x12mffwbgY0512SUHdDbJYqrXrbo9Ek0lN7wqQRsRneY2MuPzP5yHdxPk9Qfi9d92CzC5TdBtnCzNc2vj1gsKTVVhTtRjbRrVcScgLODZuTGGJVlObtXguLODkUxSuaXqrnopgH8CyAMwjYgWM/NgZl5ORG8BWAGgAsDNzFypHTYaJ9xVZ2gvAHgewCtEtA6RmcLIVOrmFL9zyFzfv739TimSrqowJ7RuVCdhnAMQn1/KbzJVMDhBjM/qzBiTOOleJpGSYGDm9wC8l2DbeADjTcqLAcSJVGY+CuDyVOrjNm77DBOd6KytVFddW+bimn7eCw6v8aPTdHMRHUHwEpUlTdMFeaos0M8iRiSZKlpPj3w19dHM28+pFoLBD4yL+fjNKT6orhKtQ+AXYbC/nd7W+0DVFrpsBAvvScZ06x8iGGwYUNgM53XJQ7MUF1cx0qud9zaGMKC6LoBXjDrHmzV39SST1yfT8FtZZQxebehz9tJUF2vymlALhmvPjIzK/zGyZ8J9XrmxL168IfHCPE7Q3/xu528XgsEuMFJIT+rW9H9AMXpgJ/ud0oRQJ9G795JuuLpvu7RMglcd8KPLFMOo90TXjhBSI6i8askQ6n88JztLhIKHdG/jr+toEIRhwqBfZc4rfPRMFxQItWDwnZDd/KOSXBPaCUEbRt2I1Uh30jXSXvAOEQw+4re7Wr2A0wz4oX8PWpXkR/yLjKYFvxHB4CPjLlZf39gNfh6wy2sY1Cx+EIYRe/Pc1NNqZAKPXHYqfhJQwkQnhNr47Df1avk7gg86+MtqIRnX0A2mB53sXm7+dCJodZkf+O0uGhQj+7TDyD7uZIT2EhEMPpLfuC5G9GyNX/mge08H/F5UXXVxlEyjY15wrs1uJEJUQbRl6YUIBh/JziI8MdKbJf8E/1RX0UXh/WJw92AW8QGC8fcXgkdsDEJGk5frfwRpq0bB6cNPM0mTXR2QWIn0Qv4NIaMhIlxRlG+/YzXhxevPCLoKnvDAcPNU1V4yoDA9VmVMR0QwCBlP1MujqKBJwDXxnsZ13Vk3IN1oXK96/q5MRRSIIWHMBYVBV8EzBhTmYdMjw4Kuhi+EwXVVCB6ZMYSETF+cXAgPzVxaTU1IHuktBEFIK+79SXdfrhP0OhfpjAgGwVXuHto16Cp4zi8CiCi/6dxOuOnczEnbnAo1fQrMvL5/AdpYLBsbZsTGUM3pkd8QS0oO+Obj36JB9U9tcOUZ7XDXO0t9vebYi/0VuL3bN8bC7/f5ek2/ISJ8/LtzcaxSouuMyIyhmtOvU1NfrxeWCNaCptU7q+rjV/QAUP3zXdXKyUb9gFcBTEekRUKCX/l2crKreU+iMfP2c1BZVX2lYJAr0/Xv7O9gRohHBIPgKkMCTN/gJ7VrBJvSvDrToHY4EuqlMyIYqjm/HtARq38oxVV92vpyvZzsLLx905loLUa9jKaRljzv8t7+3DdCeiGCoZrTrH4tTLqhj6/XPCMEEcjVndzaNbDmwYtRIySqQSGWlIzPRHQ5ES0noioiKtKVFxDRj0S0WHv9S7etNxEtJaJ1RPQkabmZiagWEb2plX9NRAWp1E0QhNSomZPle+p0IT1I1StpGYDLAMw12baemXtqr5t05c8AGAWgUHsN0cpvBLCPmTsDmADgrynWTRAEQUiClAQDM69k5tWq+xNRKwANmHkeRxayfRnACG3zcAAvaZ//A+ACkuGKIAiC73hpY+hARIsAHARwDzN/DqANgBLdPiVaGbT3LQDAzBVEdABAUwC7jScmolGIzDrQrl36L5MnCII974zuj7U7SoOuhgAFwUBEcwCY+SD+kZmnJDhsO4B2zLyHiHoD+C8RdQdMnemjzuBW22ILmScCmAgARUVF1deZXBBCRO/2jdG7feOgqyFAQTAw8yCnJ2XmMgBl2ueFRLQewEmIzBD0q6rkA9imfS4B0BZACRHlAGgIYK/TawuCIAip4UlKDCLKI6Js7XNHRIzMG5h5O4BSIuqn2Q+uBRCddUwFcJ32+WcAPtbsEIIgCIKPpOqueikRlQA4E8A0IvpQ23QOgO+IaAkihuSbmDk6+h8N4DkA6wCsBzBDK38eQFMiWgfgTgBjU6mbIAiCkByU6YPyoqIiLi4uDroagiAIGQURLWTmIrNtkl1VEARBiEEEgyAIghCDCAZBEAQhBhEMgiAIQgwZb3wmol0Avk/y8GYwiawWTJG2UkPaSQ1pJ3W8aqv2zJxntiHjBUMqEFFxIqu8EIu0lRrSTmpIO6kTRFuJKkkQBEGIQQSDIAiCEEPYBcPEoCuQQUhbqSHtpIa0kzq+t1WobQyCIAhCPGGfMQiCIAgGRDAIgiAIMYRWMBDRECJaTUTriCh0mVyJqC0RfUJEK4loORGN0cqbENFsIlqrvTfWHTNOa6/VRDRYV96biJZq256sjkuyElE2ES0iog+079JOBoioERH9h4hWaffVmdJO8RDRHdozt4yI3iCi2mnXTswcuheAbERSfncEUBPAEgDdgq6Xz23QCsDp2udcAGsAdAPwNwBjtfKxAP6qfe6mtVMtAB209svWti1AJPU6IZJG/eKgf58H7XUngNcBfKB9l3aKb6OXAPxK+1wTQCNpp7g2agNgI4A62ve3AFyfbu0U1hlDHwDrmHkDM5cDmAxgeMB18hVm3s7M32qfSwGsROSmHY7IAw7tfYT2eTiAycxcxswbEVlPow8RtQLQgJnnceRufVl3TLWAiPIBDENkHZEo0k46iKgBIuuwPA8AzFzOzPsh7WRGDoA62kqVdRFZxTKt2imsgqENgC267yVaWSghogIAvQB8DaAFR1bag/beXNstUZu10T4by6sTTwD4A4AqXZm0UywdAewC8KKmcnuOiOpB2ikGZt4K4FEAmwFsB3CAmWchzdoprILBTBcXSr9dIqoP4B0AtzPzQatdTcrYorxaQESXANjJzAtVDzEpq/bthMgo+HQAzzBzLwCHYb0KYyjbSbMdDEdELdQaQD0iusbqEJMyz9sprIKhBEBb3fd8RKZzoYKIaiAiFF5j5ne14h3aNBXa+06tPFGblWifjeXVhbMA/JSINiGicjyfiF6FtJOREgAlzPy19v0/iAgKaadYBgHYyMy7mPkYgHcB9EeatVNYBcM3AAqJqAMR1QQwEsDUgOvkK5oHw/MAVjLz47pNUwFcp32+DsAUXflIIqpFRB0AFAJYoE17S4mon3bOa3XHZDzMPI6Z85m5AJH75GNmvgbSTjEw8w8AthBRF63oAgArIO1kZDOAfkRUV/t9FyBi30uvdgraSh/UC8BQRDxx1gP4Y9D1CeD3n43I1PM7AIu111AATQF8BGCt9t5Ed8wftfZaDZ0HBIAiAMu0bf8HLaK+ur0ADMQJryRpp/j26QmgWLun/gugsbSTaTvdD2CV9htfQcTjKK3aSVJiCIIgCDGEVZUkCIIgJEAEgyAIghCDCAZBEAQhBhEMgiAIQgwiGARBEIQYRDAIgiAIMYhgEARBEGL4f+oMPlizCa/aAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b9310956",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_fft_and_scaler(data, start=5192, end=8192):\n",
    "    data = np.fft.fft(data)\n",
    "    data = np.abs(data)\n",
    "    data = data/np.expand_dims(data.max(axis=1), axis=1)\n",
    "    return data[:,start:end]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8da08bed",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sp = get_fft_and_scaler(train)\n",
    "test_sp = get_fft_and_scaler(test)\n",
    "val_sp = get_fft_and_scaler(val)\n",
    "#train_fft = np.fft.fft(train)\n",
    "#val_fft = np.fft.fft(val)\n",
    "#test_fft = np.fft.fft(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "15929181",
   "metadata": {},
   "outputs": [],
   "source": [
    "#train_abs = np.abs(train_fft)\n",
    "#val_abs = np.abs(val_fft)\n",
    "#test_abs = np.abs(test_fft)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a97c5b2c",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f905760e1f0>]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAeWUlEQVR4nO3deXxddZ3/8dcnW9O06Z7uLWmhpVaFArGAoKDsuFR/Mgo64oYMM4LLTwfqoDOoM4r7uIBYkB84PwSRRYotlkVqhdIlla6UljRNadrSJE2TJs2e+50/7km4ublJ7j25e9/PxyOP3nvOufd+vj3JO998z/ecY845REQk8+WkugAREYkPBbqISJZQoIuIZAkFuohIllCgi4hkibxUffCkSZNcaWlpqj5eRCQjbdq0qc45VxJpXcoCvbS0lPLy8lR9vIhIRjKzfQOt05CLiEiWUKCLiGQJBbqISJZQoIuIZAkFuohIlhgy0M3sXjOrMbPtA6w3M/u5mVWY2VYzOzP+ZYqIyFCi6aHfB1w+yPorgHne1/XAr4ZfloiIxGrIQHfOrQHqB9lkCfBbF7QOGGdm0+JVoEi2WV95hNcON6W6DMlC8RhDnwHsD3le7S3rx8yuN7NyMyuvra2Nw0eLZJ6PLVvHJT9dk+oyJAvFI9AtwrKId81wzi1zzpU558pKSiKeuSoiIj7FI9CrgVkhz2cCB+PwviIiEoN4BPpy4Fpvtss5QKNz7lAc3ldERGIw5MW5zOxB4EJgkplVA/8B5AM45+4CVgJXAhVAC/CZRBUrIiIDGzLQnXPXDLHeAV+IW0UiIuKLzhQVEckSCnSRFPmvFa+kugTJMgp0kRS5+297Afjx07soXbqCru5AiiuSTKdAF0mxZWsqAegKRDx9QyRqCnSRFDrQ0Ep7V7Bn7pTnMkwKdJEUOu/2v6S6BMkiCnSRJNqwd7Dr3IkMjwJdJImOtXYOur6hpYNr791AbVN7kiqSbKJAF0mi+paOAdc5HA9u2M+a3bXc80JlEquSbKFAF0mibzwe8cZfgA6KyvAp0EWSqGOQuebKcxkuBbpImnDqosswKdBFkuT3G18fdL3iXIZLgS6SJLc8um3Q9eqgy3Ap0EXShHOO+9buTXUZksEU6CJpwjk4fMybf67euvigQBdJEy9U1KW6BMlwCnSRJIhmBstND76chEokmynQRZJABzwlGRToIkmgPJdkUKCLJEFAXXRJAgW6SBLc+vjgc9BF4kGBLpIED5dXp7oEOQEo0EXSkAZoxA8FuohIllCgi4hkCQW6iEiWUKCLiGQJBbqISJZQoIukIUt1AZKRFOgiIlkiqkA3s8vNbJeZVZjZ0gjrx5rZk2a2xcx2mNln4l+qyIlD89DFjyED3cxygTuAK4CFwDVmtjBssy8ArzjnTgcuBH5sZgVxrlVERAYRTQ99MVDhnKt0znUADwFLwrZxQLGZGTAaqAe64lqpiIgMKppAnwHsD3le7S0L9UvgLcBBYBvwJedcIPyNzOx6Mys3s/La2lqfJYuISCTRBHqkA+7hQ3yXAZuB6cAi4JdmNqbfi5xb5pwrc86VlZSUxFysyIkimjsciYSLJtCrgVkhz2cS7ImH+gzwmAuqAPYCC+JTooiIRCOaQN8IzDOzOd6BzquB5WHbvA5cBGBmU4BTgcp4FipyIlEHXfzIG2oD51yXmd0IrAJygXudczvM7AZv/V3Ad4D7zGwbwSGaW5xzuoW5iE/Kc/FjyEAHcM6tBFaGLbsr5PFB4NL4liZy4lIPXfzQmaIiIllCgS6ShpwGXcQHBbqISJZQoIukIY2hix8KdBGRLKFAFxHJEgp0kTSkU//FDwW6SBpSnIsfCnQRkSyhQBcRyRIKdBGRLKFAF0lDOiYqfijQRUSyhAJdJA3pWi7ihwJdJA1pyEX8UKCLpCHlufihQBcRyRIKdJEkmDCqIKbtNeQifijQRUSyhAJdJC2piy6xU6CLiGQJBbpIGtIYuvihQBdJglivbx5QoosPCnSRNGRYqkuQDKRAFxHJEgp0EZEsoUAXEckSCnSRJIj1EGd+nsbQJXYKdJE0NHVMYapLkAykQBdJQ5q1KH5EFehmdrmZ7TKzCjNbOsA2F5rZZjPbYWZ/jW+ZIiIylLyhNjCzXOAO4BKgGthoZsudc6+EbDMOuBO43Dn3uplNTlTBIicCddDFj2h66IuBCudcpXOuA3gIWBK2zceBx5xzrwM452riW6ZIZtMQiiRDNIE+A9gf8rzaWxZqPjDezFab2SYzuzbSG5nZ9WZWbmbltbW1/ioWOQHoF4D4EU2gR5o/Ff7tlgecBbwPuAz4ppnN7/ci55Y558qcc2UlJSUxFysiIgMbcgydYI98VsjzmcDBCNvUOeeOA8fNbA1wOrA7LlWKiMiQoumhbwTmmdkcMysArgaWh23zBPAuM8szsyLgbGBnfEsVOXE4HRYVH4YMdOdcF3AjsIpgSD/snNthZjeY2Q3eNjuBPwNbgQ3APc657YkrWySzhF8+970L+k4E+8dzZiezHMlS0Qy54JxbCawMW3ZX2PMfAj+MX2ki2eu0mWNp6ehiXWV9xPU6KCp+6ExRkTSQY7p2iwyfAl0kDYTHuTro4ocCXSQFPlo2S8MqEncKdJEkCM/u6eNG9nlu4UMuSnvxQYEukiKhka0hdIkHBbpIioRPZRQZLgW6SIqE5nn4LBdFvfihQBdJkUBIomvEReJBgS6SIgEvz2dPKOKqspl91mk0RvxQoIskg4Or3zErfBEAP7t6EaMKojppW2RQCnSRJCkKC+2eg6L9piyii3OJPwp0kRTpGVbR+LnEiwJdJEV6DopGuo6LxtDFDwW6SIr0HBTVSUUSLwp0kSSI1OF+9/xJAJQUj0huMZK1dGhdJEnCe+I3X7aAT7+zlCljCqk+2tJnnUZcxA/10EVSJDfHmDY2eJGu0SPUt5LhU6CLJNH4ovyIy8cVFfDoP5/b+1wHRcUPdQtEkmjNze+hoysQcd3bZoxNcjWSbdRDF0mCnpOIigvzmTg68kHQEXm5VN3+PgrycnRikfiiQBdJkmhnJ2oWo/ilQBdJR+qgiw8KdBGRLKFAF0kzOnNU/FKgiyRBrCMoGnERPxToIkkSbc/bdFhUfFKgi6Qh3UBa/FCgi6QZjaGLXwp0kTSkDrr4oUAXSYJYAloddPFLgS6SJJHuHToQddDFj6gC3cwuN7NdZlZhZksH2e4dZtZtZlfFr0QREYnGkIFuZrnAHcAVwELgGjNbOMB23wdWxbtIkRNJLD15kVDR9NAXAxXOuUrnXAfwELAkwnY3AY8CNXGsT+SEpIOi4kc0gT4D2B/yvNpb1svMZgAfBu4a7I3M7HozKzez8tra2lhrFclYsVwOV/1z8SuaQI/0/RX+3fnfwC3Oue7B3sg5t8w5V+acKyspKYm2RpGsEEtQ63ro4kc0dyyqBmaFPJ8JHAzbpgx4yBv7mwRcaWZdzrk/xqVKkROJuujiUzSBvhGYZ2ZzgAPA1cDHQzdwzs3peWxm9wF/UpiL+KcxdPFjyEB3znWZ2Y0EZ6/kAvc653aY2Q3e+kHHzUUkNuqgi19R3STaObcSWBm2LGKQO+c+PfyyRLKLetySDDpTVCRZ1PWWBFOgi6QZnVgkfinQRdKQrocufijQRdKMOujilwJdJAl0T1FJBgW6SJJEe69QddDFLwW6SBrSELr4oUAXSTOa5SJ+KdBF0pAuziV+KNBFkkH5LEmgQBdJkmhHUjTgIn4p0EXSkA6Kih8KdJE0o2Oi4pcCXSQNqYMufijQRZIgtlkr6qKLPwp0kSSJ6Z6i6qKLDwp0kTSjMXTxS4EuIpIlFOgiaUljLhI7BbpIEsQyJq4RF/FLgS6SJLGMjeugqPihQBdJMzooKn4p0EXSkHro4ocCXSTNRHtnI5FwCnSRNKTroYsfCnSRJIjpxH910MUnBbpIkmgoRRJNgS6ShnRQVPxQoIukGfXjxS8FukgaUgdd/Igq0M3scjPbZWYVZrY0wvpPmNlW72utmZ0e/1JFMpeLYQzFdFRUfBoy0M0sF7gDuAJYCFxjZgvDNtsLXOCcOw34DrAs3oWKZDqd+i+JFk0PfTFQ4ZyrdM51AA8BS0I3cM6tdc4d9Z6uA2bGt0wRERlKNIE+A9gf8rzaWzaQzwFPRVphZtebWbmZldfW1kZfpcgJRicWiR/RBHqkPxQjfreZ2XsIBvotkdY755Y558qcc2UlJSXRVylyAtEQuviVF8U21cCskOczgYPhG5nZacA9wBXOuSPxKU8kO6i/LckQTQ99IzDPzOaYWQFwNbA8dAMzmw08BnzSObc7/mWKZC7nHM5BTkxHRRNXj2SvIXvozrkuM7sRWAXkAvc653aY2Q3e+ruAfwcmAnd6U666nHNliStbJHP0zFiJNtA15CJ+RTPkgnNuJbAybNldIY+vA66Lb2ki2SHgJXqOOuiSYDpTVCTBAl46R9vz1kW8xC8FukiC9UxBjOUM0FjOLBXpoUAXSTCNoUuyKNBFEkxj6JIsCnSRBAvE2kNPYC2S3RToIgnW00PXUIokmgJdJMFcIPhvbAdFE1SMZDUFukiC9cxyiXYMXddDF78U6CIJFusYOuigqPijQBdJsNbObgAaWjqj2l79c/FLgS6SYAcbWgFo6+qO+jU6sUj8UKCLJFhPj/vcuRNje4FIjBToIgnW0R2c5pKfG/2Pm/rn4ocCXSTBOruD8VyQF92Pmzro4pcCXSTBOruCPfSCGHroIn7oO0wkwXqHXPJ0MRdJLAW6SIJ1xjiGrhOLxC8FukiCdfgYcnHqoosPCnSRBOsZctFBUUk0BbpIgvUcFI1p2uIJ3EH/+N3rKPvPZ1NdRkaK6ibRIuJPIOC4Y/UeIIYe+gneRV+750iqS8hY6qGLJNDdf6uktqkdgPzc6JM6cCJ30cU3BbpIAty2fAdf+f1m/rCpundZfk50P24TR41g1Y7D/O21WgD+uruWh8v3J6TOcBU1Tb2/gFKtp/0SPQW6SBw1tHRw3u1/4b61VTz+8gEqapp71+VEeUH0MSODI6Gf/M0GAD517wZufmRr/IuN4OKfrOH87/8l7u/7zCuHaWjpiOk1Pe2X6CnQRYA7nq9g+ZaDUW1bWdvMGd9+mgPeVRRDvVhxJOLyWMRy8DTUC6/V8c7vPUcg4Hhi8wFKl66gpqkt5vdp9w7i+rVpXz1vNL75ufXHO/j8b8u5/rebhvW+MjQFuqSV5VsO+g4igDca23ixoi7m1/1w1S6++ODLvb3IhpYOjrVFvn75Qxv3c7Slkyc2H+hd1tLRxZrdtRxqHF6YA9SEDHm898erex/vr2/pfXysrZPm9i4aWzspXbqCnz37Gv/4m/UcbGzjnhcq+d361wHYU3N8yM+77v6NfOvJHYNu09kdoLM7ENVlfT/yq5e4+Cd/BWD34Sa6vGmblXXNA77GOUcgEPm932hsy/jhl1889xqlS1ck/LLImuUiaeWBdfsAL4gcVDe0MiIvh7mTRnPP3yq54cKTycsxfrhqFx86YwbzpxT3ef37f/E36po7qLr9fVTVHWf93iMsWTSDwvxcXqyoY09tM8+/WsMtVyxgwdQx/T5/0befoer297Ho288A8JWL5/Oli+cB0NzeRa5Z7w/lD/68iwVTi/nd+v2MK8rnkZDx8uEIHZiprH0zkD/yq7U885ULGJGfw2m3Pd3nNT99dnfv4+dfre09Lemau9cBMH1sIZ9/91w+c96cfp/37M4aAL566al9lgcCjtbObt76H6t6l80YN5InbzqfFyrqaGzt5AOnTWNcUUG/92xu72LD3no++uuX+OJ7TwGgO0JgP7+rhpnjRnLrH7ezYW89Vbe/r8/6f/3Dlt7jEN/7P29n7qRRzJ9SzPhR/T8znf34meD+6Qq4mA6Ox0qBLmmlpwNjBu/6wfO9f/5/+p2l3Le2io7uAKfNHMedq/fwx5cPsPbrF+Gc4+ZHtvY5AHn3mkr+a+VOAPYdaeHKt0/jE/es711/rK2LK98+jVOnFHP6rLF9aihduqL38U+f3c25J0/kpIlFnP3d5/rV+9n7ygFYMLW43zq/xo7Mj7i8pqmd07/9dMR1oV6qPNLvrNSDjW1868lX6A44rnvXXCDY655361O927wtJLgfWL+P//diVZ9jAAAHGlr57H0b2by/AYC/7qrllx8/gwXf/DOLZo3rXQ6wpzb42p//pQKAoy2dtHd1MyIvl4aWDr72h608u/Nwn/dfcseLfZ6H7tOvP7at93F48GeK9q6A7yG1aCjQJa1sqKoH4KfP7O4zlnvf2ioAfuGFAwRD6rcvVZGXk9PnBx/oDXOAO1fv4U5vLniPTfuOsmnf0ahq+uivXxpym+b2rqjeKxrjI/R4Y9Vzdmq4/1yxk4aWTi56y2Q+fOfaAV9/6+PbB1wXGtrP7jzMqh1v9FsOsLeu/3DPvS9UsWTRdN55e+QDr1vC3mMgX314CxecWsIF80sYmZ8b9Rz/VGvv7Gb0iMTFrqXqVldlZWWuvLw8JZ8t6Wf7gUZufXwbW6obU11KQjzxhfM4fda4qLZtbO3kuvs3srEqul84kv499p6/+l76+nuZNnbksN7LzDY558oircuMX2uSVjq7A70XnBqu9ZVHKF26gvf/4oWsDXMg6jCH4JDL7z5/TgKryT6Lvv00H797HcvW7KGzOzDgAdZk2n6gkdKlK/r85dLeGZ+fm4FEFehmdrmZ7TKzCjNbGmG9mdnPvfVbzezM+Jcau2NtnRxv7+JgQytHj3ewsaqeJm/mgt9ZFALv+dFqTvvWqqE3HMAbjW2c9Z1nWHLHi3xs2bo4VpY9EjnOmo0aWjpZu+cI3135KvNufYq5/7aS827/Cw9tCM72WbO7lsdfDg7LLfjmU3zvqZ19Xv/sK4f55h/7DzPtqW3GOUd3wPV2YnYfbqKuue/JVz3bhFrjzcxZsfXN6bDDnRI6lCGHXMwsF9gNXAJUAxuBa5xzr4RscyVwE3AlcDbwM+fc2YO973CGXDbsrWfltkP8Q9lM2joDVNUdZ+b4kYwuzKOqroWpYwv54oMvRz0f+MJTS7ju/Lksmj2O51+toXTiKA40tPJ6/XG6vB354TNmcKChlcnFhbzR2MbeumYWzRpPU1snz++qYfq4kZw0sYiaY+2sqzzC/qOtXHXWTCaOKmD/0VY+cPo0DjW00dDaye83vs5FC6ZQf7yDZ3Ye5up3zKKxtZMdB4/x4TNm0NEVoKapjUPeXN6TJhbR2eXodo4jze2UThrFrPFFtHV2s2FvPVPGFPLIpmpOnVrM6/UtjMjL4V3zS6hv7uD0WWNpbu8iPzeH//v7zSyaPY4PLZpBY2sne+uOk5+bw643mjjU2Mo337+Qz93/5j4pyM3pHYu9YH4Jja2d/cZJeyyeM4FpYwt5YvNBzGDamEKmji2kuDCf0olFPFxeTWtn9He9T2dnnTQ+6vF3gPlTRvP0Vy6I+XM6ugJs2ne0d6YKwIcWTeePm/vOl//apfP50dO7w18uaWLa2MLen+Ue3/rgW/nUO0t9vd9gQy7RBPq5wG3Oucu8518HcM59L2SbXwOrnXMPes93ARc65w4N9L5+A/3JLQe56cGXY36dZKZHbjiXvXXHCTjHLY9u45TJo/vNvOjx6D+fy4KpY/pMswv11Uvm808XnMz8bzzFRQsmc83i2bxQUccD6/f13vcT4M5PnMm/PPB3AL500Tx+9txrveuevPF83jZjDGbGqh1v8E//s4mR+bm9v6zOO2UiL1Yc4d5Pl1FWOoFXDzXxlmnFFBdGnrkSje+u3MmyNZU89aV3MWFUQZ/ZNlv+/VLGFuXz1921PP9qDUsWTWduyWiOt3ex9LFtfPD06XztD1v6vef4onxmTShi6wDDXFe8bSo/uOo0vv/nV/n/617vXX7/ZxczbWwhq3fV0NzWxe7DzfzZOyh67bknUXOsnQ8ums6/PPB3vnbpfH61eg/HO/r+Il9cOoHxo/JZteMwpROLqDry5vz6bbddykd/vY6dh46x/t8uorzqKKt31fQe9P7yxfP472ff3B8fK5vFn7Ye7PcZ6e668+fwjfcv9PXa4Qb6VcDlzrnrvOefBM52zt0Yss2fgNudcy94z58DbnHOlYe91/XA9QCzZ88+a9++fTE3ZuehY1y9bB2NrZFP+oDgnNsvXzKf/Fyjqa2LCaMKqKo7zrvnl3D4WDunTB7N6/Ut7DtynDca23pnQLyjdPyQB6JmjBtJ6aQiXqw4wvwpo6mqaxlwRkGo4sI8mtq6KMjLGdb4c0nxCGZPKGLTvqPk5ljEub0AHzh9OjsONDK6MK/fD+1bpo1h56FjUX/mpNEjcM5RUjyCooJcqo+29jn5BWBy8Yg+y0bk5TB5zAgaWjppautixriRTBxdwGVvncqYwjzW7jnCTe+dx8LpY2jt6KauuZ2O7gAnl4wesp7Wjm72H21hSnEhHd0BJnpzksNPrd9YVc+zrxzmpImjmDNpFOeePBGgdx75UHcG2l/fwszxI2nvCnCstZPJYwojbuec4/ldNVw4fzJm8FpNc7/58fF09HgH44ryfd3ZaPP+BkqKRzBjXPDAXGNLJ/l5xoGjrTy55SDXX3Byv1kYNU1tjCrIo+rIcd46ve8Uz0DAsea1WmZPKGLuAPuurbObprYutlY3UJCXw+I5ExiRl8tx7y/HTfuOYgaLZo2jMD+XQMDRFXB9Zq50dAXIyzFycowjze38/fUGLlk4pXd9XXM7G/bWs3DaGB4u38/owjwWl07AzPjRql1cddZMzjtlUu+MnJEFudz8yFZOmljEzPEjWVdZz6lTisnPNbZUN3LeKRM566QJjB2Zz8zxIynIy6G+uYMt1Q1sP9BIS0c3r77RxMVvmczUsYWs3XOEj5w5k+b2Li5aMJkfrNrFv152KrsPN/GH8mqOtXbywOfPprgwnyPN7cyeUOT7zlTDDfR/AC4LC/TFzrmbQrZZAXwvLNBvds4NeK6vZrmIiMRuuLNcqoFZIc9nAuEXvYhmGxERSaBoAn0jMM/M5phZAXA1sDxsm+XAtd5sl3OAxsHGz0VEJP6GPGXJOddlZjcCq4Bc4F7n3A4zu8FbfxewkuAMlwqgBfhM4koWEZFIojoH1Tm3kmBohy67K+SxA74Q39JERCQWOntBRCRLKNBFRLKEAl1EJEso0EVEskTKLp9rZrVA7KeKBk0CYr/PWHpSW9JTtrQlW9oBakuPk5xzJZFWpCzQh8PMygc6UyrTqC3pKVvaki3tALUlGhpyERHJEgp0EZEskamBvizVBcSR2pKesqUt2dIOUFuGlJFj6CIi0l+m9tBFRCSMAl1EJEtkXKAPdcPqdGNmVWa2zcw2m1m5t2yCmT1jZq95/44P2f7rXtt2mdllqasczOxeM6sxs+0hy2Ku3czO8v4PKrybifu7VUv823KbmR3w9s1m7964ad0WM5tlZs+b2U4z22FmX/KWZ9x+GaQtmbhfCs1sg5lt8dryLW95cveLcy5jvghevncPMBcoALYAC1Nd1xA1VwGTwpb9AFjqPV4KfN97vNBr0whgjtfW3BTW/m7gTGD7cGoHNgDnAgY8BVyRJm25DfhahG3Tti3ANOBM73ExwRu4L8zE/TJIWzJxvxgw2nucD6wHzkn2fsm0HvpioMI5V+mc6wAeApakuCY/lgD3e4/vBz4Usvwh51y7c24vwevLL05BfQA459YA9WGLY6rdzKYBY5xzL7ngd+tvQ16TNAO0ZSBp2xbn3CHn3N+9x03ATmAGGbhfBmnLQNK5Lc4513P38nzvy5Hk/ZJpgT4D2B/yvJrBvwHSgQOeNrNNFrxJNsAU593Ryft3src8E9oXa+0zvMfhy9PFjWa21RuS6flzOCPaYmalwBkEe4MZvV/C2gIZuF/MLNfMNgM1wDPOuaTvl0wL9EhjSek+7/I859yZwBXAF8zs3YNsm4nt6zFQ7encpl8BJwOLgEPAj73lad8WMxsNPAp82Tl3bLBNIyxL97Zk5H5xznU75xYRvKfyYjN72yCbJ6QtmRboGXczaufcQe/fGuBxgkMoh70/rfD+rfE2z4T2xVp7tfc4fHnKOecOez+EAeBu3hzeSuu2mFk+wQB8wDn3mLc4I/dLpLZk6n7p4ZxrAFYDl5Pk/ZJpgR7NDavThpmNMrPinsfApcB2gjV/ytvsU8AT3uPlwNVmNsLM5gDzCB4gSScx1e79mdlkZud4R+uvDXlNSvX8oHk+THDfQBq3xfvc3wA7nXM/CVmVcftloLZk6H4pMbNx3uORwMXAqyR7vyTzSHA8vgjejHo3waPCt6a6niFqnUvwSPYWYEdPvcBE4DngNe/fCSGvudVr2y5SMBskrP4HCf7J20mw5/A5P7UDZQR/KPcAv8Q7QzkN2vI/wDZgq/cDNi3d2wKcT/BP8K3AZu/rykzcL4O0JRP3y2nAy17N24F/95Yndb/o1H8RkSyRaUMuIiIyAAW6iEiWUKCLiGQJBbqISJZQoIuIZAkFuohIllCgi4hkif8FmY10ujcnYkUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAfR0lEQVR4nO3de3xcZb3v8c8vSdNCL5a2oZbe0kKhdG+RllDuCCpSQF9s1K2ARy5eeHGOsGGf4z5U3SAeOEcUUbYiAkJFVCiiKAVa7pdSKA2pvSW9hvSS9JKkTUkvaS6Tec4fM0knySSZWZ1ZM7Pyfb9eeXXNmidrfquTfPPMs561ljnnEBGR3JeX6QJERCQ1FOgiIgGhQBcRCQgFuohIQCjQRUQCoiBTLzxmzBhXXFycqZcXEclJy5cv3+2cK4r3XMYCvbi4mLKysky9vIhITjKzrb09pyEXEZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJiH4D3czmmVmdmZX38ryZ2S/NrNLMVpvZrNSXKSIi/Umkh/44MKeP5y8BpkW/bgB+c+RliYhIsvoNdOfcYqChjyaXA0+4iPeBkWY2LlUFigTVcyu3c6AllOkyJEBSMYY+HqiOeVwTXdeDmd1gZmVmVlZfX5+ClxbJTeXbG7ll/krm/nV1pkuRAElFoFucdXHvmuGce8Q5V+KcKykqinvmqsiAcKitHYBdjc0ZrkSCJBWBXgNMjHk8AdiRgu2KBJ7uFyaplIpAXwBcE53tcibQ6JzbmYLtigRWx8da3QJSUqnfi3OZ2VPABcAYM6sBfggMAnDOPQQsBC4FKoEm4Pp0FSsSFBZNdMW5pFK/ge6cu6qf5x3wnZRVJBJQ4bBjW0MTxWOG0tFHVwddUklnior45JdvbOKCn71FZd0B9dAlLRToIj4p3Rw5naN2X3PcqWEiR0qBLuKzLsMsGnORFFKgi/jELHY5OoaeoVokmBToIhlweNpiRsuQgFGgi2TA4YOiSnRJHQW6iE/erdwDRELcdFhU0kCBLpIBr66rBTTkIqmlQBfJgF++vglQoEtqKdBFMkh5LqmkQBfxwUvl8a9Xp4tzSSop0EV8cMv8lZ3LynBJFwW6SAbtPtCS6RIkQBToIhm0+0Arb66vy3QZEhAKdJEMu/7xDzJdggSEAl3EB7HXcblmXmnmCpFAU6CL+KC/A6GfuPNlbpm/wp9iJLAU6CJZYH9ziOdW6t7qcmQU6CIiAaFAF/FBSyic6RJkAFCgi4gEhAJdRCQgFOgiIgGhQBfJIks27c50CZLDFOgiWeS/PbYs0yVIDlOgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCIqFAN7M5ZrbBzCrNbG6c5z9mZs+b2SozqzCz61NfqoiI9KXfQDezfODXwCXADOAqM5vRrdl3gLXOuU8CFwD3mVlhimsVEZE+JNJDnw1UOueqnHOtwHzg8m5tHDDczAwYBjQAoZRWKiIifUok0McD1TGPa6LrYj0AnAzsANYAtzjnelwv1MxuMLMyMyurr6/3WLKIiMSTSKBbnHXdb6h1MbASOA44FXjAzEb0+CbnHnHOlTjnSoqKipIuVkREepdIoNcAE2MeTyDSE491PfCsi6gENgPTU1OiyMByqLUd199NSEXiSCTQPwCmmdmU6IHOK4EF3dpsAz4DYGZjgZOAqlQWKjJQnHzHS/zx/a2ZLkNyUL+B7pwLATcBLwPrgD875yrM7EYzuzHa7C7gbDNbA7wO3Oac03VARTxaVL4r0yVIDipIpJFzbiGwsNu6h2KWdwCfS21pIgOXRlzEC50pKiISEAp0EZGAUKCLZCHXY2awSP8U6CIiAaFAFxEJCAW6iEhAKNBFspCmLYoXCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EWykGYtihcKdBGRgFCgi4gEhAJdJBtpzEU8UKCLiASEAl1EJCAU6CIiAaFAF8lCumOReKFAFxEJCAW6iEhAKNBFRAJCgS6ShXTHIvFCgS4iEhAKdJEsZJbpCiQXKdBFRAJCgS4iEhAKdBGRgEgo0M1sjpltMLNKM5vbS5sLzGylmVWY2dupLVNkYDE0iC7JK+ivgZnlA78GLgJqgA/MbIFzbm1Mm5HAg8Ac59w2Mzs2XQWLiEh8ifTQZwOVzrkq51wrMB+4vFubq4FnnXPbAJxzdaktU0RE+pNIoI8HqmMe10TXxToROMbM3jKz5WZ2TbwNmdkNZlZmZmX19fXeKhYZCDTiIh4kEujxfrS6n8dWAJwGXAZcDNxuZif2+CbnHnHOlTjnSoqKipIuVmTA0Jmi4kG/Y+hEeuQTYx5PAHbEabPbOXcQOGhmi4FPAhtTUqXIAKPL54oXifTQPwCmmdkUMysErgQWdGvzHHCemRWY2dHAGcC61JYqMnDoWi7iRb89dOdcyMxuAl4G8oF5zrkKM7sx+vxDzrl1ZvYSsBoIA48658rTWbhIkCnPxYtEhlxwzi0EFnZb91C3x/cC96auNJGBy6mLLh7oTFERH100Y2xC7RTn4oUCXcRHU4uGZroECTAFuogPJo46isEFeXz3cycl1F4jLuKFAl3EJ5d+YhyD8hP7lVOeixcKdBGfJHPypw6KihcKdBGf/elbZ2S6BAkoBbqID2I73OecMKZzefzIo/ptL5IoBbqIX+KMubz1HxfEbapT/8ULBbpIBvV2kFQ9dPFCgS6SJY45elDnsgJdvFCgi/ggkYBeccfnuPCkyGWllefihQJdxCeJ3CfULNJG0xbFCwW6iEhAKNBFskhHH14ddPFCgS7iE0vgVNGONpq2KF4o0EWykHro4oUCXcQHOsgpflCgi/gkqYtzpa0KCTIFukhWicR+WD168UCBLuKDROO588Cp8lw8UKCL+CSRWS4iR6Ig0wWIDHR/ufEs9jW3dVmnDrp4oUAX8UFfQ+IlxaM6lw+fWKRIl+RpyEXEJ4lcy6WD4ly8UKCLZJHOM0WV6OKBAl3EB4meyt/Ri9ep/+KFAl3EJ8nMclEPXbxQoIuIBIQCXcQH8XrcX5w5Pqn2Iv3RtEURn8QOuWy557I+22jaonihHrpIFpk8eigAxwwtzHAlkosSCnQzm2NmG8ys0szm9tHudDNrN7Mvp65EkYHjf33uRADOnDo6w5VILuo30M0sH/g1cAkwA7jKzGb00u4nwMupLlIk1yU6gDIoP49hgws0hi6eJNJDnw1UOueqnHOtwHzg8jjtbgb+CtSlsD6RAEls3qKu4SVeJRLo44HqmMc10XWdzGw8cAXwUF8bMrMbzKzMzMrq6+uTrVVkwNCJReJFIoEer8PQ/aftfuA251x7Xxtyzj3inCtxzpUUFRUlWqNIzktqCEVddPEokWmLNcDEmMcTgB3d2pQA8y0y52oMcKmZhZxzf09JlSIBoOuhS7olEugfANPMbAqwHbgSuDq2gXNuSseymT0OvKAwF/FOB0XFi34D3TkXMrObiMxeyQfmOecqzOzG6PN9jpuLCCRzQVx15MWrhM4Udc4tBBZ2Wxc3yJ1z1x15WSLBo6CWdNOZoiJZxjTYLh4p0EV8kOyYuK7lIl4o0EV8kmjHWx108UqBLpKF1D8XLxToIj7QeUXiBwW6iE8siajWELp4oUAXyTKa5SJeKdBFfJDsrBVdnEu8UKCL+CThWS7pLUMCTIEukoU0hi5eKNBFfJDULBd10cUjBbqIT5TTkm4KdJEspBEX8UKBLpJ11JcXbxToIj5I/uJc6alDgk2BLuKTRE8Y0kFR8UqBLpKV1EWX5CnQRXyQzJmi6qCLVwp0kSykMXTxQoEukmU0hi5eKdBFfJBsh1s9dPFCgS7ik8QvzqUuunijQBfJQrp8rnihQBfxQxL5rDF08UqBLuIT3YJO0k2BLpJl1EEXrxToIj5Qh1v8oEAX8UkyY+P6AyBeKNBFskyiF/ES6U6BLuKDZK7lEmmfpkIk0BIKdDObY2YbzKzSzObGef5rZrY6+vWemX0y9aWK5Db1uyXd+g10M8sHfg1cAswArjKzGd2abQY+5Zw7BbgLeCTVhYoMJDqxSLxIpIc+G6h0zlU551qB+cDlsQ2cc+855/ZGH74PTEhtmSK5LZl41hC6eJVIoI8HqmMe10TX9eabwKJ4T5jZDWZWZmZl9fX1iVcpEgBJBbU66OJBIoEe78cw7o+bmV1IJNBvi/e8c+4R51yJc66kqKgo8SpFBhD10MWrggTa1AATYx5PAHZ0b2RmpwCPApc45/akpjyRgUkddPEikR76B8A0M5tiZoXAlcCC2AZmNgl4Fvi6c25j6ssUyW3JTEPU5XPFq3576M65kJndBLwM5APznHMVZnZj9PmHgDuA0cCD0ZMiQs65kvSVLZJ7kjlhKNl56yKQ2JALzrmFwMJu6x6KWf4W8K3UliYyMGkMXbzSmaIiPkh2Xrn65+KFAl3EJ4l2vNVBF68U6CIiAaFAF/FBssc4dUxUvFCgi/glwbEUXT5XvFKgi2QhddDFCwW6iA+SujhX2qqQoFOgi/gkmTNAdWKReKFAF8k26qKLRwp0ET8kO8slPVVIwCnQRXyS6OQVddDFKwW6SBaqaWiiqv5ApsuQHKNAF/FBMtdyMTNW1TTy6fveTmNFEkQKdBGfaChF0k2BLpJlFPzilQJdxAeaVi5+UKCLZBldykW8UqCL+CTRoN5Yq9kt4o0CXUSyyu/e3cym2v2ZLiMnKdBFfJDMEPqcf/p42uroz48XreO5ldsz9voAP3p+LZf9cklGa0iFcNjxUvkuwmH/DqAo0EVS5NW1tWzY1XvPMtGLcw0ZlLlfy4ffruKW+St5pWIXxXNfpLqhydfX77goWWt72NfXTYdnlldz4x+X82TpNt9eU4EuOcE5R/n2xoy+/p0LKlixbW+vbb79RBkX37+4y7r2sEu6hza4ID+hds1t7dw6fwW7GpuT2n4i/vqPGgAqdhz5//lXH15K8dwXE2rb7mNvNt1q97UApOX96Y0CXXLCU6XVfP5XS7jxD8t578PdR7StFdv29np52vawY/HG+h7rW0JhHn9vC199+P1+t79k0+H6TvjBQqbf/hLtYcc7m3puN56DraHO5YaDrb22e21dLX9fuYO7Xlib0HaT0fHf80xZzRFfynfZ5oboNh0NB1vZ8dGhXtu2d3st5yL/b9v7+J5slRf9QBb2cc6qAl1ywoZd+wB4qWIXV/92meftvLq2lisefI+nSqtZvLGe0mjYdPjtO1VcM6+U/3hmFZ/9+ds9e4wJjJrc8Vx557Jzh4cPVtUk1tt9YfXOzuVXKnb12i4vOm2mIzBeW1vb4/ovm3cfZPnW3j9VdNi652CXXnTHbr++vo431tclVHd//lxWzay7XuXse96IO06//aNDXDuvtMu6Bat28PXHSjnnnjdYXfNRSurwS15ex/vj42v691Ii3v1+6dZ+2zS3tbO/uQ2AP7y/lW17mtjZeIjiuS+y9MM9QCS4AL7/tzVcM6+Urzy8lC27D3ZuoyMQn1leQ2XdATbVRcbEX1tXC0BrKMwLq3f0eO3KusNj50f6+zt7yqjO5bnPruGxJZt79JKrG5r4H3/6BxAJdOcc33qirMf1Xy782Vt86TfvdT7efaCF+v0tzC/d1uWP1Vsbun56WL718B+6Ay0hKuv2d1nXXXvYsWDVjj6Hl27765rO5Vvmr+S630WCurmtHYD7XtnA+1VdX2NnzHBFVX3kfWpsauv1NRJx1SPv8+Sy9I9rd/zB9fNmJQW+vZJIkkLtYU74wSLu/MKMuM9vrN3PCUXDWL29kSmjh/L5B96huuEQG+6ew+1/L2fsiMFcefokAP7w/hbOOn503I+/F9+/mA13XxJ3nHfO/e+w7v/M4aYnV3Suu+nJFV0en338aN6L/sGASMCWb2/kiaVbPO33BScVdfnkcNcLa1m0ZidfPm0Ctfta+MVrG7u0b2t3PP1BdefjUHtkeOjEscM715VubuBHz1dQsWNf57p25/jaGZPZe7CVHy6o6LLNvd1C87M/jxwb2HLPZUBkbH3GuBG0hMIU5ufx+HtbuOuFtTS1hLhy9qTO77v8gd5nq3T8EZl++0s8dm0JB5pDXZ7/8aJ1tIYOHxy99emV3Pr0SgCeufEsTi8exf2vbeRQazvfu/TkXl+nu6VVe1hatYerz5jUf+MjkIkhFwW6JK26oYlQ2DFlzNC0bL+tPUxTSzsW/fz4s1c29mjz/Kod3PzUCq49a3KP3vtJ//kSEDko9V+vbwJg4ZpdLN+6t0vwdmgJhfs8aHfFg+/2WW/3bYad4/O/8j7tLt5smLKteynrZejkjW7DIvPe3cz/W7i+S5uvPLy0x/ftjY7Pz7zr1T7refy9LZ3L4bDjhwsq+MP7W7n50yfwqzcqu7Sd++waHl5cxZvfvQBIfJjpm78v67Hu4berem3/6tpaTjx2OPe/Fnl/kwl0vxweEvPvNRXokrTzfvomcLi3lkqL1uzkv0eHEmYXR4YeDrSEerS7ZX6kh5zIUEyH2KGHZKzvYypiPNUN8Q/gXXd2cULfn3eEp/53D/PePPx2Fd+58IR+263Ydnjseur3F3Yudw/zDpt3H0z7bJVHFlfxyOLDgb+sag8HW0N8evrYznWNTW08tPhD/udFJzIo3//RZet2jMMPCnTJGjs+OtQZ5gClW3ofs83F2W3fPn9qQu3yfLqYy/6WEI++szkt2z4+Jvj98NVHIrOPvjhzPB8dauP6c4pZVL6LJ5dt46Sxw/mXmeP7/P4319dRvr2Rmz8zLWU1dQ65hB3VDU2UbW3gipkTUrb9uK+Z1q2LJCAcdjy2ZDNn3/NGpktJq8EFif26XTj92DRXctj/XbjOt9fyw7MrtvPG+jq+/lgpbdHx95ZQO6+treXuPqZ3Xv/4B9z3amRor7qhif3NbYTawzy3crvng5qxQy5XPPge//70qrQfIFUPXTLmqdJtHGwJcfeLwQqV3gwZlNgJQyccOyzNlQwMzyyPnBy1fOte/lwWWf5+zFh7zd4mzv3Jmyz8t/M617WHHef99E1OHDuMr5RM5O4X19EednxxVs+edXNbO48t2cy3z5tKYZw/1rEHRXcfiJxk1NbuKCxI3yewhLoMZjbHzDaYWaWZzY3zvJnZL6PPrzazWakvNXnbPzrEY0s28+SybbSE2tlYu59r5pV2nu33buVufvhcOd97dnW/22oNhand10xTa6jzzK/2sOPD+gO8UrGL5rb2uGcy1u1vpq097GlMsW5/8xH1EDocbAlxqLW9x/rmtnZ2NvZ+wkbs626s3c+t81ewOWaKX1/zgsu3N3YedAuHHbsam9m65yCLN9bz5vo6/vxBNd97ds2ACXOAwiTGccePPCqNlQwsHWEOcNEvDk/rvCg6c+f3MQd9O07q2lh7gD3Rn9+djc08tmQzn7nvLQCaWkO0tYd59J0q7n15A0+VbqPhYGvnlNgO1Xsjv1uxB6zb2sPsbDxEKE2XNrD+wsLM8oGNwEVADfABcJVzbm1Mm0uBm4FLgTOA/3LOndHXdktKSlxZWc8j2/1ZsW0vVzzo7eBWf86cOqrHPFjJHd84Zwrz3j3yMeFxHxvSZf5zPMOHFLC/uefB2t7ccP5UvnfJ9M4DZf3507Kt/OBv5T3WFw0fTP3+loRf9/bPz0jLmaSJevLbZ/Q4ESzezKRE3fPFT1C6pYFn/5HZC4gdqfNPLOKJb8z29L1mttw5VxLvuUSGXGYDlc65qujG5gOXA7E/JZcDT7jIX4f3zWykmY1zzu3subkj03F9hHRQmGeHwoI8rju7uMsshg5jhhVy5emTeODNys62HXOV7/jCDO74wgw21e7nol9Eel+/u+50qnYf5GBLiMmjj2b51r08sXQrZx8/mvNPLOKeRV1nhPzma7M4+/gx7G1qpTg6LbN0cwN/WV7Np6ePpaT4GO5cUME9XzqFX72+iYe71fhPx42gYsc+7v3yKfzHXyKf/L557pQuH/UT8bUzJnPuCWP41L1vda67bc50Lj/1uLjHGk6dOJJrz57Mvz+9qnPdo9eU8NkZY/nyaRNYVrWHTXUHmDFuBPcsWs+4kUN4a0M9/3nZyZw5dTTTPz6cW59eyaihhSyrauCFfzuXXY3NHDO0kNZQmFnRqY3TPz6ca84q5vt/W8O5J4zh1IkjaQuHKZk8irOOH82KbXsp376Pq2dPYlCBdR68vvMLM7junCmdtU0bO5z7XtnAvuYQG+++hFfX1nLutDHsajzEWxvq+cY5U2hoaqXk7td46ttnMnPSSAryjIL8PEqKR/HsP7bzqROLeDvmMg0zJ43sMiMnm40Ykp7R7kR66F8G5jjnvhV9/HXgDOfcTTFtXgDucc4tiT5+HbjNOVfWbVs3ADcATJo06bStW5P/K+2c42uPLos7n3jGuBGs3bmPq2ZPZNKooVx+6nG8VL6LowrzWbhmJ1fPnsSufc3ML63mvGljOGfaGJ5ftYOXy3cxbEgBuw+0dhkaOXncCEYPLWTFtr0cjBmymDFuBMeOGMzpxaNYsmk3S6t61tLhvGljqNvXwqzJIynMz6Mqeir2tGOHdc7RPW/aGN6Juf7HZ6YfS1NrO0ur9vD1MydTv7+FnY2HmDx6KOeeMIbnV++gIM94c0M9o4cWMnPSMbSE2rtsY9akkRxqC7Nu5z5OnRj5ZVi/a3/nFMDxI4/islPG0Rq9RgnAyKMHUTRsMF86bQKraz5i4qijefjtKgryjFDYcfK4EazbuY/bPz+DUyeO5C/La3gq5kpy554whiWVuzv/76Z/fDh/W3G4J3XZKeM4dcJI/rVkAjsbmzl53Ajq9jVz7Igh7D7QwvAhBYTDkJ9n5OcZm3cfoCAvj6MK82lua2fByh1cd04xw4cM6vxZAGhqbScUdnzsqEGdr7Wq+iNmHDci4elqHWeYdmw7GYvW7GTk0YWcdfzoHs8dbAkxZFA++R7nIja3tXOgJcSYYYM7122q3U9LKMw/j/8YlXUHyDOYWnR43D0cdoTCLu64bgfnHGFHwnW1tYcpyLOEP2Gk277mNkYMGcT6Xfs45uhCxo4Y0uX5VdUf0RIKs2LbXj4x/mMUDR/MyKML2dbQxKRRR3Pvy+u57JTjqG1sZt67mznp48MZlJ/Hup37mDlpJH98fxtHF+Zz8rgRnZdOuPasyby2rq7zujKnThzJ1j0H2dvURmF+HhdOL2JT3QHa2sNUNxzihZvPZdrYYfzi1U2Ub29k6OB8Xq6o5bJTxvHAVTM9/1/21UNPJND/Fbi4W6DPds7dHNPmReDH3QL9fzvnlve2Xa9DLiIiA1lfgZ5I96UGmBjzeALQ/WIWibQREZE0SiTQPwCmmdkUMysErgQWdGuzALgmOtvlTKAxHePnIiLSu35H5p1zITO7CXgZyAfmOecqzOzG6PMPAQuJzHCpBJqA69NXsoiIxJPQoVbn3EIioR277qGYZQd8J7WliYhIMnTqv4hIQCjQRUQCQoEuIhIQCnQRkYDo98SitL2wWT3g7YIOMAY4slu/Zw/tS3YKyr4EZT9A+9JhsnOuKN4TGQv0I2FmZb2dKZVrtC/ZKSj7EpT9AO1LIjTkIiISEAp0EZGAyNVAfyTTBaSQ9iU7BWVfgrIfoH3pV06OoYuISE+52kMXEZFuFOgiIgGRc4He3w2rs42ZbTGzNWa20szKoutGmdmrZrYp+u8xMe2/F923DWZ2ceYqBzObZ2Z1ZlYesy7p2s3stOj/QWX0ZuK+3/aml32508y2R9+bldF742b1vpjZRDN708zWmVmFmd0SXZ9z70sf+5KL78sQMys1s1XRfflRdL2/74tzLme+iFy+90NgKlAIrAJmZLqufmreAozptu6nwNzo8lzgJ9HlGdF9GgxMie5rfgZrPx+YBZQfSe1AKXAWYMAi4JIs2Zc7ge/GaZu1+wKMA2ZFl4cTuYH7jFx8X/rYl1x8XwwYFl0eBCwDzvT7fcm1HnrnDaudc61Axw2rc83lwO+jy78H/iVm/XznXItzbjOR68t7uzV4CjjnFgPd75ydVO1mNg4Y4Zxb6iI/rU/EfI9vetmX3mTtvjjndjrn/hFd3g+sA8aTg+9LH/vSm2zeF+ecOxB9OCj65fD5fcm1QB8PVMc8rqHvH4Bs4IBXzGy5RW6SDTDWRe/oFP332Oj6XNi/ZGsfH13uvj5b3GRmq6NDMh0fh3NiX8ysGJhJpDeY0+9Lt32BHHxfzCzfzFYCdcCrzjnf35dcC/R4Y0nZPu/yHOfcLOAS4Dtmdn4fbXNx/zr0Vns279NvgOOBU4GdwH3R9Vm/L2Y2DPgrcKtzbl9fTeOsy/Z9ycn3xTnX7pw7lcg9lWeb2T/30Twt+5JrgZ5zN6N2zu2I/lsH/I3IEEpt9KMV0X/ros1zYf+Srb0mutx9fcY552qjv4Rh4LccHt7K6n0xs0FEAvBPzrlno6tz8n2Jty+5+r50cM59BLwFzMHn9yXXAj2RG1ZnDTMbambDO5aBzwHlRGq+NtrsWuC56PIC4EozG2xmU4BpRA6QZJOkao9+zNxvZmdGj9ZfE/M9GdXxixZ1BZH3BrJ4X6Kv+xiwzjn385incu596W1fcvR9KTKzkdHlo4DPAuvx+33x80hwKr6I3Ix6I5Gjwj/IdD391DqVyJHsVUBFR73AaOB1YFP031Ex3/OD6L5tIAOzQbrV/xSRj7xtRHoO3/RSO1BC5JfyQ+ABomcoZ8G+/AFYA6yO/oKNy/Z9Ac4l8hF8NbAy+nVpLr4vfexLLr4vpwArojWXA3dE1/v6vujUfxGRgMi1IRcREemFAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhD/HxKpCEh0DZQgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plt.plot(train_sp[0])\n",
    "plt.figure()\n",
    "plt.plot(val_sp[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9c68101",
   "metadata": {},
   "source": [
    "# DNN建模分析"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0bb21c03",
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'tqdm'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-17-b767689ec8db>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# dnn模型构建\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtqdm\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'tqdm'"
     ]
    }
   ],
   "source": [
    "# dnn模型构建\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import TensorDataset, DataLoader, Dataset\n",
    "import torch.nn.functional as F\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb9b5e0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 固定随机数种子，确保实验的可重复性\n",
    "def set_seed(seed=42):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "\n",
    "set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5805c6b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 搭建DNN模型\n",
    "class DNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.dnn = nn.Sequential(\n",
    "            nn.BatchNorm1d(3000),\n",
    "            nn.Linear(3000, 512),\n",
    "            nn.BatchNorm1d(512),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(p=0.2),\n",
    "            nn.Linear(512,256),\n",
    "            nn.BatchNorm1d(256),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(p=0.2),\n",
    "            nn.Linear(256, 10),\n",
    "        )\n",
    "    def forward(self, x):\n",
    "        x = self.dnn(x)\n",
    "        return F.softmax(x, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c2130834",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4257\n",
      "10081\n",
      "4270\n",
      "17187\n",
      "4746\n",
      "6337\n",
      "10298\n",
      "3694\n",
      "5058\n",
      "4281\n"
     ]
    }
   ],
   "source": [
    "for i in Counter(train_label).values():\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a8a2ef56",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将数据转换成pytorch的tensor\n",
    "batch_size = 128\n",
    "\n",
    "train_tensor = torch.tensor(train_sp).float()\n",
    "y_train_tensor = torch.tensor(train_label).long()\n",
    "val_tensor = torch.tensor(val_sp).float()\n",
    "y_val_tensor = torch.tensor(val_label).long()\n",
    "\n",
    "# 使用Dataloader对数据进行封装\n",
    "train_tensor = TensorDataset(train_tensor, y_train_tensor)\n",
    "val_tensor = TensorDataset(val_tensor, y_val_tensor)\n",
    "\n",
    "\n",
    "train_loader = DataLoader(train_tensor, shuffle=True, batch_size=batch_size, drop_last=True)\n",
    "val_loader = DataLoader(val_tensor, shuffle=False, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "d9d159a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 0.0001\n",
    "gamma = 1\n",
    "step_size = 5\n",
    "EPOCH = 80\n",
    "\n",
    "model = DNN().cuda()\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "lf = nn.CrossEntropyLoss()\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=gamma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "fe20f01b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DNN(\n",
      "  (dnn): Sequential(\n",
      "    (0): BatchNorm1d(3000, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (1): Linear(in_features=3000, out_features=512, bias=True)\n",
      "    (2): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (3): ReLU()\n",
      "    (4): Dropout(p=0.2, inplace=False)\n",
      "    (5): Linear(in_features=512, out_features=256, bias=True)\n",
      "    (6): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (7): ReLU()\n",
      "    (8): Dropout(p=0.2, inplace=False)\n",
      "    (9): Linear(in_features=256, out_features=10, bias=True)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "18756b8c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 151.59it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 383.17it/s]\n",
      "  3%|▎         | 15/548 [00:00<00:03, 141.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0 loss:1.810878 train_acc:0.549317\n",
      "val_acc:0.493911\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 160.79it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 447.25it/s]\n",
      "  2%|▏         | 12/548 [00:00<00:04, 114.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1 loss:1.819555 train_acc:0.659901\n",
      "val_acc:0.498739\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:04<00:00, 120.52it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 436.95it/s]\n",
      "  3%|▎         | 16/548 [00:00<00:03, 152.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 2 loss:1.798020 train_acc:0.694085\n",
      "val_acc:0.491347\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:04<00:00, 123.23it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 371.75it/s]\n",
      "  2%|▏         | 12/548 [00:00<00:04, 113.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 3 loss:1.783508 train_acc:0.721218\n",
      "val_acc:0.481135\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:04<00:00, 131.93it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 441.58it/s]\n",
      "  3%|▎         | 16/548 [00:00<00:03, 151.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 4 loss:1.759122 train_acc:0.739777\n",
      "val_acc:0.492202\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 161.97it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 390.03it/s]\n",
      "  2%|▏         | 13/548 [00:00<00:04, 129.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 5 loss:1.771167 train_acc:0.756698\n",
      "val_acc:0.479938\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 148.41it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 456.52it/s]\n",
      "  3%|▎         | 15/548 [00:00<00:03, 148.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 6 loss:1.662324 train_acc:0.776382\n",
      "val_acc:0.482630\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 161.89it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 462.79it/s]\n",
      "  2%|▏         | 13/548 [00:00<00:04, 128.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 7 loss:1.688476 train_acc:0.795681\n",
      "val_acc:0.494766\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 159.76it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 390.90it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:04, 133.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 8 loss:1.691044 train_acc:0.805467\n",
      "val_acc:0.505106\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:04<00:00, 135.71it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 419.16it/s]\n",
      "  3%|▎         | 19/548 [00:00<00:02, 181.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 9 loss:1.668591 train_acc:0.815280\n",
      "val_acc:0.487630\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 142.10it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 441.98it/s]\n",
      "  3%|▎         | 18/548 [00:00<00:03, 172.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:10 loss:1.657607 train_acc:0.821120\n",
      "val_acc:0.504593\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 158.28it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 378.27it/s]\n",
      "  2%|▏         | 13/548 [00:00<00:04, 125.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:11 loss:1.639763 train_acc:0.826960\n",
      "val_acc:0.500961\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:04<00:00, 121.26it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 458.04it/s]\n",
      "  3%|▎         | 16/548 [00:00<00:03, 157.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:12 loss:1.651890 train_acc:0.831674\n",
      "val_acc:0.496389\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:04<00:00, 119.54it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 375.85it/s]\n",
      "  3%|▎         | 15/548 [00:00<00:03, 140.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:13 loss:1.591116 train_acc:0.837272\n",
      "val_acc:0.488570\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 146.55it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 420.26it/s]\n",
      "  3%|▎         | 16/548 [00:00<00:03, 154.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:14 loss:1.632923 train_acc:0.842456\n",
      "val_acc:0.507627\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 151.82it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 412.69it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 137.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:15 loss:1.628713 train_acc:0.844934\n",
      "val_acc:0.501218\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 139.86it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 463.28it/s]\n",
      "  3%|▎         | 16/548 [00:00<00:03, 152.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:16 loss:1.618376 train_acc:0.848780\n",
      "val_acc:0.491817\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|████▊     | 267/548 [00:01<00:01, 163.87it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m--------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                        Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-66-2284e356782a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0mlabel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m         \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeature\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    887\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 889\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    890\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    891\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-61-b61a1ca9bc38>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     16\u001b[0m         )\n\u001b[1;32m     17\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdnn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    887\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 889\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    890\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    891\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    117\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 119\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    120\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    121\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    887\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 889\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    890\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    891\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/nn/modules/linear.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 94\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlinear\u001b[0;34m(input, weight, bias)\u001b[0m\n\u001b[1;32m   1751\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhas_torch_function_variadic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1752\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mhandle_torch_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1753\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_nn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1754\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1755\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "train_best = 1000\n",
    "for epoch in range(EPOCH):\n",
    "    model.train()\n",
    "    train_total_acc = 0\n",
    "    \n",
    "    for feature, label in tqdm(train_loader):\n",
    "        feature = feature.cuda()\n",
    "        label = label.cuda()\n",
    "        optimizer.zero_grad()\n",
    "        preds = model(feature)\n",
    "        loss = lf(preds, label)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        train_total_acc += model(feature).argmax(dim=1).eq(label).sum().item()\n",
    "        \n",
    "        feature.cpu()\n",
    "        label.cpu()\n",
    "    \n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        val_total_acc = 0\n",
    "        for feature, label in tqdm(val_loader):\n",
    "            feature = feature.cuda()\n",
    "            label = label.cuda()\n",
    "            val_total_acc += model(feature).argmax(dim=1).eq(label).sum().item()\n",
    "            \n",
    "            feature.cpu()\n",
    "            label.cpu()\n",
    "    \n",
    "    scheduler.step()\n",
    "    print(f'epoch:{epoch:2} loss:{loss:4f} train_acc:{train_total_acc/len(train_label):4f}')\n",
    "    print('val_acc:{:4f}'.format(val_total_acc/len(val_label)))\n",
    "    \n",
    "    # 保存最佳的训练模型\n",
    "    if loss <= train_best:\n",
    "        train_best = loss\n",
    "        torch.save(model.state_dict(), './model.point')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "26737f7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.3997)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model(torch.tensor(val).float().cuda()).cpu().argmax(dim=1).eq(torch.tensor(val_label)).sum()/len(val_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5bada4e8",
   "metadata": {},
   "source": [
    "# Xgboost模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e69dd29e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "# https://github.com/dmlc/xgboost/tree/master/demo\n",
    "# https://github.com/dmlc/xgboost/blob/master/demo/gpu_acceleration/cover_type.py\n",
    "# https://github.com/dmlc/xgboost/blob/master/demo/multiclass_classification/train.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e9793cf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "xg_train = xgb.DMatrix(train_sp, label=train_label)\n",
    "xg_val = xgb.DMatrix(val_sp, label=val_label)\n",
    "xg_test = xgb.DMatrix(test_sp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "79348cab",
   "metadata": {},
   "outputs": [],
   "source": [
    "# setup parameters for xgboost\n",
    "param = {}\n",
    "# use softmax multi-class classification\n",
    "param['objective'] = 'multi:softmax'\n",
    "# scale weight of positive examples\n",
    "param['eta'] = 0.1\n",
    "param['max_depth'] = 6\n",
    "param['nthread'] = -1\n",
    "param['num_class'] = 10\n",
    "param['tree_method']: 'gpu_hist'\n",
    "param['eval_metric'] = 'mlogloss'\n",
    "#watchlist = [(xg_train, 'train'), (xg_test, 'test')]\n",
    "num_round = 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "45879f11",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test acc using softmax = 0.44135367260607616\n"
     ]
    }
   ],
   "source": [
    "bst = xgb.train(param, xg_train, num_round)\n",
    "# get prediction\n",
    "pred = bst.predict(xg_val)\n",
    "acc_rate = np.sum(pred == val_label) / val_label.shape[0]\n",
    "print('Test acc using softmax = {}'.format(acc_rate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07d3e537",
   "metadata": {},
   "outputs": [],
   "source": [
    "ans = model.predict(x_test)\n",
    "np.save(ans, 'xgboost_preds.npy')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b0618c5",
   "metadata": {},
   "source": [
    "# Catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "44d9a9fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from catboost import Pool, CatBoostClassifier\n",
    "# 参考：https://github.com/catboost/catboost/blob/master/catboost/tutorials/classification/classification_tutorial.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "c41057e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CatBoostClassifier(#loss_function=\"Logloss\",\n",
    "                           eval_metric=\"AUC\",\n",
    "                           task_type=\"GPU\",\n",
    "                           learning_rate=0.1,\n",
    "                           iterations=2000,\n",
    "                           l2_leaf_reg=50,\n",
    "                           random_seed=43,\n",
    "                           od_type=\"Iter\",\n",
    "                           depth=5,\n",
    "                           early_stopping_rounds=15000,\n",
    "                           border_count=64,\n",
    "                           loss_function='MultiClass',\n",
    "                           #has_time= True \n",
    "                          )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "5dec7816",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#model.fit(\n",
    "#    train_sp, train_label,\n",
    "#    #eval_set=(val_sp, val_label),\n",
    "#)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "d1cd0c6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = model.predict(val_sp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "8712a350",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.484254155450156"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(preds.flatten() == val_label).sum() / len(val_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "31f1d917",
   "metadata": {},
   "outputs": [],
   "source": [
    "ans = model.predict(test_sp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "891715c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1]\n",
      " [3]\n",
      " [9]\n",
      " ...\n",
      " [8]\n",
      " [3]\n",
      " [3]]\n"
     ]
    }
   ],
   "source": [
    "print(ans)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b60a9854",
   "metadata": {},
   "source": [
    "# LSTM模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "75ef4eb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision\n",
    "from tqdm import tqdm\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "from torch.utils.data import TensorDataset, DataLoader, Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "c78f3661",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "\n",
    "train_tensor = torch.tensor(train_sp).float()\n",
    "y_train_tensor = torch.tensor(train_label).long()\n",
    "val_tensor = torch.tensor(val_sp).float()\n",
    "y_val_tensor = torch.tensor(val_label).long()\n",
    "\n",
    "# 使用Dataloader对数据进行封装\n",
    "train_tensor = TensorDataset(train_tensor, y_train_tensor)\n",
    "val_tensor = TensorDataset(val_tensor, y_val_tensor)\n",
    "\n",
    "\n",
    "train_loader = DataLoader(train_tensor, shuffle=True, batch_size=batch_size, drop_last=True)\n",
    "val_loader = DataLoader(val_tensor, shuffle=False, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "7286472c",
   "metadata": {},
   "outputs": [],
   "source": [
    "sequence_length = 1\n",
    "input_size = 3000\n",
    "hidden_size = 512\n",
    "num_layers = 2\n",
    "num_classes = 10\n",
    "batch_size = 100\n",
    "num_epochs = 30\n",
    "learning_rate = 0.01\n",
    "gamma = 0.9\n",
    "step_size=1\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "3b6e213d",
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNN(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, num_layers, num_classes):\n",
    "        super(RNN, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        # 官网默认的输入是seq_len, batch, input_size,这里将batch first=True放在第一个\n",
    "        # seq len 相当于一句话的长度， input size相当于每个单词的向量长度\n",
    "        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_size, num_classes)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # 初始化两个向量的值, 因为这里是两层的lstm\n",
    "        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)\n",
    "        c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)\n",
    "\n",
    "        # 前向传播，这里只需要out即可\n",
    "        out, _ = self.lstm(x, (h0, c0))  # out: tensor of shape (batch_size, seq_length, hidden_size)\n",
    "\n",
    "        # 使用全连层进行分类\n",
    "        out = self.fc(out[:, -1, :])\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "8f64eee8",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = RNN(input_size, hidden_size, num_layers, num_classes).to(device)\n",
    "# Loss and optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=gamma) # 学习方式\n",
    "# Train the model\n",
    "total_step = len(train_loader)\n",
    "train_acc, val_acc = [], []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "595fa7a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 138.72it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 435.00it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 138.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 0, train loss:1.5093, train acc:0.5071\n",
      "val acc:0.4722\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 141.26it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 437.75it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 138.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 1, train loss:0.9834, train acc:0.6626\n",
      "val acc:0.5351\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 141.93it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 443.02it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 138.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 2, train loss:0.8539, train acc:0.7011\n",
      "val acc:0.5221\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 141.91it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 441.73it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 138.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 3, train loss:0.7638, train acc:0.7316\n",
      "val acc:0.5341\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 141.14it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 430.77it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 138.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 4, train loss:0.7109, train acc:0.7504\n",
      "val acc:0.5175\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 147.25it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 439.37it/s]\n",
      "  3%|▎         | 16/548 [00:00<00:03, 151.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 5, train loss:0.6657, train acc:0.7653\n",
      "val acc:0.4767\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 152.65it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 439.69it/s]\n",
      "  3%|▎         | 16/548 [00:00<00:03, 150.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 6, train loss:0.6236, train acc:0.7812\n",
      "val acc:0.5093\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 152.88it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 436.91it/s]\n",
      "  3%|▎         | 15/548 [00:00<00:03, 149.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 7, train loss:0.5843, train acc:0.7948\n",
      "val acc:0.5088\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 152.97it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 436.60it/s]\n",
      "  3%|▎         | 16/548 [00:00<00:03, 151.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 8, train loss:0.5521, train acc:0.8059\n",
      "val acc:0.5222\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 152.82it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 438.69it/s]\n",
      "  3%|▎         | 16/548 [00:00<00:03, 150.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 9, train loss:0.5164, train acc:0.8188\n",
      "val acc:0.5004\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 152.84it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 439.35it/s]\n",
      "  3%|▎         | 16/548 [00:00<00:03, 150.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:10, train loss:0.4945, train acc:0.8250\n",
      "val acc:0.5269\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 152.58it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 436.51it/s]\n",
      "  3%|▎         | 16/548 [00:00<00:03, 150.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:11, train loss:0.4648, train acc:0.8353\n",
      "val acc:0.5279\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 152.65it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 438.99it/s]\n",
      "  3%|▎         | 16/548 [00:00<00:03, 150.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:12, train loss:0.4454, train acc:0.8404\n",
      "val acc:0.5057\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 141.50it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 437.59it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 137.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:13, train loss:0.4230, train acc:0.8497\n",
      "val acc:0.5101\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 139.70it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 373.20it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 134.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:14, train loss:0.4050, train acc:0.8562\n",
      "val acc:0.5277\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 137.78it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 346.20it/s]\n",
      "  3%|▎         | 15/548 [00:00<00:03, 145.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:15, train loss:0.3901, train acc:0.8615\n",
      "val acc:0.5180\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 138.08it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 432.76it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 134.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:16, train loss:0.3725, train acc:0.8679\n",
      "val acc:0.4915\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 138.71it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 439.56it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 137.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:17, train loss:0.3583, train acc:0.8719\n",
      "val acc:0.4951\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 138.17it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 439.12it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 134.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:18, train loss:0.3443, train acc:0.8768\n",
      "val acc:0.5268\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 137.71it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 435.96it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 133.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:19, train loss:0.3347, train acc:0.8797\n",
      "val acc:0.4979\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 139.65it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 441.00it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 139.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:20, train loss:0.3225, train acc:0.8854\n",
      "val acc:0.5034\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 140.67it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 440.70it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 135.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:21, train loss:0.3118, train acc:0.8894\n",
      "val acc:0.5089\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 142.38it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 438.60it/s]\n",
      "  3%|▎         | 15/548 [00:00<00:03, 142.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:22, train loss:0.3042, train acc:0.8927\n",
      "val acc:0.5104\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 141.16it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 443.98it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 138.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:23, train loss:0.2955, train acc:0.8949\n",
      "val acc:0.5106\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 142.32it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 442.78it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 138.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:24, train loss:0.2886, train acc:0.8969\n",
      "val acc:0.4917\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 141.90it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 433.86it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:04, 131.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:25, train loss:0.2807, train acc:0.9011\n",
      "val acc:0.5143\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 141.22it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 428.40it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 138.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:26, train loss:0.2733, train acc:0.9036\n",
      "val acc:0.5237\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 141.69it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 428.32it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 137.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:27, train loss:0.2687, train acc:0.9048\n",
      "val acc:0.4992\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:03<00:00, 141.78it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 255.43it/s]\n",
      "  3%|▎         | 14/548 [00:00<00:03, 137.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:28, train loss:0.2622, train acc:0.9079\n",
      "val acc:0.4980\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:04<00:00, 136.98it/s]\n",
      "100%|██████████| 183/183 [00:00<00:00, 441.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:29, train loss:0.2566, train acc:0.9092\n",
      "val acc:0.5052\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(num_epochs):\n",
    "    epoch_accuracy = 0\n",
    "    epoch_loss = 0\n",
    "    model.train()\n",
    "    for i, (images, labels) in enumerate(tqdm(train_loader)):\n",
    "        # reshape的维度变成了(batch size, 28, 28)\n",
    "        images = images.reshape(-1, sequence_length, input_size).to(device)\n",
    "        labels = labels.to(device)\n",
    "        output = model(images)\n",
    "        loss = criterion(output, labels)\n",
    "        # Backward and optimize\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        acc = (output.argmax(dim=1) == labels).float().mean()\n",
    "        epoch_accuracy += acc / len(train_loader)\n",
    "        epoch_loss += loss / len(train_loader)\n",
    "\n",
    "    # val the model\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        epoch_val_accuracy = 0\n",
    "        epoch_val_loss = 0\n",
    "        for images, labels in tqdm(val_loader):\n",
    "            images = images.reshape(-1, sequence_length, input_size).to(device)\n",
    "            labels = labels.to(device)\n",
    "            outputs = model(images)\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            acc = (predicted == labels).float().mean()\n",
    "            epoch_val_accuracy += acc / len(val_loader)\n",
    "\n",
    "    scheduler.step()\n",
    "    print(f'EPOCH:{epoch:2}, train loss:{epoch_loss:.4f}, train acc:{epoch_accuracy:.4f}')\n",
    "    print(f'val acc:{epoch_val_accuracy:.4f}')\n",
    "\n",
    "    train_acc.append(epoch_accuracy)\n",
    "    val_acc.append(epoch_val_accuracy)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b472fabd",
   "metadata": {},
   "source": [
    "# Transformer分析"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "13c78456",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader, Dataset, TensorDataset\n",
    "import torch\n",
    "import math\n",
    "from torchvision import datasets, transforms\n",
    "# 这里使用Fashion minist数据集，使用transformer进行特征提取\n",
    "# 使用linear层进行下游任务分类\n",
    "# 参考：https://github.com/oliverguhr/transformer-time-series-prediction/blob/master/transformer-multistep.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "7e01b947",
   "metadata": {},
   "outputs": [],
   "source": [
    "class PositionalEncoding(nn.Module):\n",
    "    def __init__(self, d_model=3000, max_len=1):\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "        pe = torch.zeros(max_len, d_model)\n",
    "        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n",
    "        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n",
    "        pe[:, 0::2] = torch.sin(position * div_term)\n",
    "        pe[:, 1::2] = torch.cos(position * div_term)\n",
    "        pe = pe.unsqueeze(0).transpose(0, 1)\n",
    "        self.register_buffer('pe', pe)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return x + self.pe[:x.size(0), :]\n",
    "\n",
    "# 这里feature size必须可以整除以nhead\n",
    "class Transformer(nn.Module):\n",
    "    def __init__(self, src_len=1, d_model=3000, num_layers=4, nhead=8):\n",
    "        super().__init__()\n",
    "        self.src_mask = None\n",
    "        self.pos_encoder = PositionalEncoding(d_model).to(device)\n",
    "        self.nhead = nhead\n",
    "        self.d_model = d_model\n",
    "        self.src_len = src_len\n",
    "        self.encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=self.nhead)\n",
    "        self.transformer_encoder = nn.TransformerEncoder(self.encoder_layer, num_layers=num_layers)\n",
    "        # 这里encoder输出的维度还是[batch size, max_len, embedding dim]\n",
    "        # 所以embedding dim维度应该是 max len * embedding dim\n",
    "        self.decoder = nn.Linear(self.d_model*self.src_len, 10)\n",
    "        self.init_weights()\n",
    "\n",
    "    def forward(self, x):\n",
    "        #if self.src_mask is None:\n",
    "        #    mask = self._generate_square_subsequent_mask(len(x)).to(device)\n",
    "        #    self.src_mask = mask\n",
    "        # 输入x加上位置编码后，得到encoder的输入\n",
    "        x = self.pos_encoder(x)\n",
    "        # 这里mask不是要免除padding=0后，softmax的影响吗\n",
    "        # encoder mask讲解参考下面链接\n",
    "        # https://medium.com/data-scientists-playground/transformer-encoder-mask%E7%AF%87-dc2c3abfe2e\n",
    "        # 这里不需要为encoder添加mask，首先每个长度都是一样\n",
    "        # 什么时候需要？参考：https://github.com/pytorch/tutorials/blob/master/beginner_source/transformer_tutorial.py\n",
    "        # 大致意思是说当我们要做seq2seq任务时，encoder只能接受前面位置的mask，要对后面位置的seq进行mask。\n",
    "        # 当输入的batch size 长度不一样时候，也需要mask\n",
    "        x = self.transformer_encoder(x)#, self.src_mask)\n",
    "        x = self.decoder(x.permute(1, 0, 2).reshape(-1, self.d_model*self.src_len))\n",
    "        return x\n",
    "\n",
    "    # 将decoder 的偏置转换成0， 将权重进行标准化\n",
    "    def init_weights(self):\n",
    "        initrange = 0.1\n",
    "        self.decoder.bias.data.zero_()\n",
    "        self.decoder.weight.data.uniform_(-initrange, initrange)\n",
    "\n",
    "    # 将输入的序列进行mask操作\n",
    "    def _generate_square_subsequent_mask(self, sz):\n",
    "        mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)\n",
    "        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))\n",
    "        return mask"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c92a365",
   "metadata": {},
   "source": [
    "## 数据转换"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "e9281f2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "bd868882",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_tensor = torch.tensor(train_sp).float()\n",
    "y_train_tensor = torch.tensor(train_label).long()\n",
    "val_tensor = torch.tensor(val_sp).float()\n",
    "y_val_tensor = torch.tensor(val_label).long()\n",
    "\n",
    "# 使用Dataloader对数据进行封装\n",
    "batch_size = 128\n",
    "train_tensor = TensorDataset(train_tensor, y_train_tensor)\n",
    "val_tensor = TensorDataset(val_tensor, y_val_tensor)\n",
    "\n",
    "\n",
    "train_loader = DataLoader(train_tensor, shuffle=True, batch_size=batch_size, drop_last=True)\n",
    "val_loader = DataLoader(val_tensor, shuffle=False, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "3f7db81a",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "sequence_length = 1\n",
    "# input_size 相当于embedding dim\n",
    "model = Transformer().to(device)\n",
    "input_size = 3000\n",
    "num_epochs = 80\n",
    "learning_rate = 1e-4\n",
    "gamma = 0.9\n",
    "step_size=1\n",
    "\n",
    "# Loss and optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=gamma) # 学习方式\n",
    "\n",
    "# Train the model\n",
    "total_step = len(train_loader)\n",
    "train_acc, val_acc = [], []\n",
    "# seq_len 相当于 max len， 即每句话切分的最大长度\n",
    "\n",
    "train_best = None\n",
    "best_loss = 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "1135e41e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.87it/s]\n",
      "  0%|          | 2/548 [00:00<00:38, 14.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 0, train loss:3.0662, train acc:0.1697\n",
      "val acc:0.2455\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.93it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.85it/s]\n",
      "  0%|          | 2/548 [00:00<00:40, 13.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 1, train loss:2.4007, train acc:0.1802\n",
      "val acc:0.1452\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.93it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.92it/s]\n",
      "  0%|          | 2/548 [00:00<00:38, 14.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 2, train loss:2.3348, train acc:0.1894\n",
      "val acc:0.2455\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.93it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.90it/s]\n",
      "  0%|          | 2/548 [00:00<00:36, 15.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 3, train loss:2.2458, train acc:0.2243\n",
      "val acc:0.3387\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.89it/s]\n",
      "  0%|          | 2/548 [00:00<00:37, 14.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 4, train loss:1.9161, train acc:0.3803\n",
      "val acc:0.4727\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.86it/s]\n",
      "  0%|          | 2/548 [00:00<00:37, 14.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 5, train loss:1.7717, train acc:0.4390\n",
      "val acc:0.4702\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.90it/s]\n",
      "  0%|          | 2/548 [00:00<00:36, 14.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 6, train loss:1.7033, train acc:0.4643\n",
      "val acc:0.4720\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.93it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.88it/s]\n",
      "  0%|          | 0/548 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 7, train loss:1.5680, train acc:0.4995\n",
      "val acc:0.4541\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.89it/s]\n",
      "  0%|          | 0/548 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 8, train loss:1.4367, train acc:0.5313\n",
      "val acc:0.4917\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.91it/s]\n",
      "  0%|          | 2/548 [00:00<00:36, 14.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 9, train loss:1.3241, train acc:0.5605\n",
      "val acc:0.4959\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.93it/s]\n",
      "  0%|          | 2/548 [00:00<00:37, 14.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:10, train loss:1.2374, train acc:0.5846\n",
      "val acc:0.4958\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.86it/s]\n",
      "  0%|          | 2/548 [00:00<00:37, 14.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:11, train loss:1.1666, train acc:0.6057\n",
      "val acc:0.4897\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.90it/s]\n",
      "  0%|          | 0/548 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:12, train loss:1.1173, train acc:0.6206\n",
      "val acc:0.4679\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.89it/s]\n",
      "  0%|          | 2/548 [00:00<00:37, 14.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:13, train loss:1.0729, train acc:0.6364\n",
      "val acc:0.4841\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.91it/s]\n",
      "  0%|          | 2/548 [00:00<00:37, 14.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:14, train loss:1.0426, train acc:0.6453\n",
      "val acc:0.5018\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.89it/s]\n",
      "  0%|          | 2/548 [00:00<00:38, 14.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:15, train loss:0.9937, train acc:0.6612\n",
      "val acc:0.4865\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.93it/s]\n",
      "  0%|          | 2/548 [00:00<00:37, 14.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:16, train loss:0.9568, train acc:0.6746\n",
      "val acc:0.5256\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.88it/s]\n",
      "  0%|          | 2/548 [00:00<00:37, 14.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:17, train loss:0.9261, train acc:0.6822\n",
      "val acc:0.4929\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.93it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.82it/s]\n",
      "  0%|          | 2/548 [00:00<00:36, 14.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:18, train loss:0.9007, train acc:0.6935\n",
      "val acc:0.5005\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.85it/s]\n",
      "  0%|          | 0/548 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:19, train loss:0.8887, train acc:0.6955\n",
      "val acc:0.4814\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.93it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.90it/s]\n",
      "  0%|          | 2/548 [00:00<00:36, 14.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:20, train loss:0.8581, train acc:0.7045\n",
      "val acc:0.5018\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.90it/s]\n",
      "  0%|          | 2/548 [00:00<00:37, 14.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:21, train loss:0.8459, train acc:0.7087\n",
      "val acc:0.4829\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.76it/s]\n",
      "  0%|          | 2/548 [00:00<00:37, 14.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:22, train loss:0.8260, train acc:0.7135\n",
      "val acc:0.4768\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.90it/s]\n",
      "  0%|          | 2/548 [00:00<00:36, 14.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:23, train loss:0.8170, train acc:0.7168\n",
      "val acc:0.4852\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.89it/s]\n",
      "  0%|          | 2/548 [00:00<00:36, 15.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:24, train loss:0.8046, train acc:0.7207\n",
      "val acc:0.5077\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [01:01<00:00,  8.94it/s]\n",
      "100%|██████████| 183/183 [00:03<00:00, 49.92it/s]\n",
      "  0%|          | 2/548 [00:00<00:37, 14.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:25, train loss:0.7930, train acc:0.7253\n",
      "val acc:0.5002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 439/548 [00:49<00:12,  8.92it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m--------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                        Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-92-6179a0efe5e8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m         \u001b[0;31m# reshape的维度变成了(batch size, 28, 28)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0mimages\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msequence_length\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m         \u001b[0mimages\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpermute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m         \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for epoch in range(num_epochs):\n",
    "    epoch_accuracy = 0\n",
    "    epoch_loss = 0\n",
    "    model.train()\n",
    "    for i, (images, labels) in enumerate(tqdm(train_loader)):\n",
    "        # reshape的维度变成了(batch size, 28, 28)\n",
    "        images = images.reshape(-1, sequence_length, input_size)\n",
    "        images = images.permute(1, 0, 2).to(device)\n",
    "        labels = labels.to(device)\n",
    "\n",
    "        # Forward pass\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "\n",
    "        # Backward and optimize\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        acc = (outputs.argmax(dim=1) == labels).float().mean()\n",
    "        epoch_accuracy += acc / len(train_loader)\n",
    "        epoch_loss += loss / len(train_loader)\n",
    "\n",
    "    # val the model\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        epoch_val_accuracy = 0\n",
    "        epoch_val_loss = 0\n",
    "        for images, labels in tqdm(val_loader):\n",
    "            images = images.reshape(-1, sequence_length, input_size)\n",
    "            images = images.permute(1, 0, 2).to(device)\n",
    "            labels = labels.to(device)\n",
    "            outputs = model(images)\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            acc = (predicted == labels).float().mean()\n",
    "            epoch_val_accuracy += acc / len(val_loader)\n",
    "\n",
    "    if loss <= best_loss:\n",
    "        best_model = model\n",
    "        best_loss = loss\n",
    "\n",
    "    scheduler.step()\n",
    "    print(f'EPOCH:{epoch:2}, train loss:{epoch_loss:.4f}, train acc:{epoch_accuracy:.4f}')\n",
    "    print(f'val acc:{epoch_val_accuracy:.4f}')\n",
    "\n",
    "    train_acc.append(epoch_accuracy)\n",
    "    val_acc.append(epoch_val_accuracy)\n",
    "\n",
    "# 保存模型\n",
    "torch.save(best_model.state_dict(), './transformer_model.point')\n",
    "\n",
    "# 读取模型参数\n",
    "# model.load_state_dict(torch.load('transformer_model.point'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6665a37",
   "metadata": {},
   "source": [
    "## 模型保存部分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "70615ea6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 保存模型权重\n",
    "#torch.save(model.state_dict(), './model.point')\n",
    "# 加载模型权重\n",
    "model.load_state_dict(torch.load('model.point'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31cad521",
   "metadata": {},
   "source": [
    "## 生成预测结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "32acc31a",
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = model(torch.tensor(test).float().cuda()).argmax(dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "5818eb60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([23403])\n"
     ]
    }
   ],
   "source": [
    "print(preds.shape), print(test,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "3b8f2513",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 保存预测结果, 注意提交preds.npy文件\n",
    "np.save('preds.npy', preds.cpu().numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6b80ae7",
   "metadata": {},
   "source": [
    "# Resnet模型"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e32ab72",
   "metadata": {},
   "source": [
    "## 数据处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "c38fe5b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision\n",
    "from tqdm import tqdm\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "from torch.utils.data import TensorDataset, DataLoader, Dataset\n",
    "from scipy import signal\n",
    "import gc\n",
    "from torchvision import models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "5a53ed2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sp = get_fft_and_scaler(train)\n",
    "test_sp = get_fft_and_scaler(test)\n",
    "val_sp = get_fft_and_scaler(val)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37101144",
   "metadata": {},
   "source": [
    "## STFT画图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "7bfebaf5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(126, 126)\n"
     ]
    }
   ],
   "source": [
    "# 调高noverlap y轴会变大，调低窗口数量，y轴会变小\n",
    "f, t, zxx = signal.stft(train_sp[0], fs=122.68e6, nfft=126, window=signal.get_window('hann', 30), noverlap=6, nperseg=30, return_onesided=False)\n",
    "print(zxx.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "5a12a122",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "e6fea8e5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAEDCAYAAAAcI05xAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO29abBd13Ue+O073/vumye8AXgASHAeRIoaIzmS4sQanCi204lcTlfblsNyVaau7iq306lUdcpJJXZncrqc7mZ5iFWJpcRSO4mSSG63ZVlSNJIURZEiCRAz8PDm6c7j7h94oLG+tYB7QeFRF6z1VbGIc94+++y9zz77nvXtb60VYoxwOBwOx+Ai8YNugMPhcDhuDl+oHQ6HY8DhC7XD4XAMOHyhdjgcjgGHL9QOh8Mx4PCF2uFwOAYcB7ZQhxB+M4SwFkJ4oY+y/zyE8Nz+fydDCDsH1S6Hw+G40xAOSkcdQvghAGUAn4gxPnQL1/1NAI/FGH/2QBrmcDgcdxgO7Is6xvglAFvXnwsh3BVC+HwI4ZkQwpdDCPcZl/4kgE8eVLscDofjTkPqDb7fUwB+PsZ4KoTwDgD/CsAHrv0xhLAE4BiAL7zB7XI4HI6BxRu2UIcQigDeDeB3QwjXTmep2McAfDrG2Hmj2uVwOByDjjfyizoBYCfG+JablPkYgL/+BrXH4XA47gi8YfK8GOMegLMhhP8OAMJVPHrt7yGEewGMA/jaG9Umh8PhuBNwkPK8T+LqontvCOFSCOHjAH4KwMdDCN8B8CKAj153yU8C+FT0cH4Oh8MhcGDyPIfD4XDcHrhnosPhcAw4DmQzMZPIxXxiWJxrjefEcUceAgACaT1C1yhDBkB6q67KNKdk5TGoIki0uGJdppOhe1V1mXZeHsekca9m73tF+slMNnq3B9BjxPX0C253oq3LpFcq4rg5N6TKJKmv3bRxr8TNjwH9fKI1U6nvPDesuq051TXq5n5Yc0g9R8s4pTLBGNcuPVc1NwGkN2viuDGb14X4XkZf+Tkna7oMt8caH/PdpHNWP7gu65n181yTdVmoOaonEb9D1vtjvYuMdEk2IKb0RTzPrW4l6dl36JrW7hbatYrZogNZqPOJYbxr9MfEudWfkL4tuyf0dZk92UZrEvHgz3/yFVXm/M/eK4554gFA4QoNvrHAlpbk8ezTenauv0VOkNaILlM8JyvvsCgRQHtItmfktC5TmdfPMEW/U62ivo5/AK3J2RqW989s60KL/+ir4vjiz71blRm6LOupHtL1cF87OT2t86tyXJtjukyiIeu2Fga+V8p4D+rT+pkNn5X3tz4s1I+AJSqlMtkt3Y/yYdmm/Jqu5tBvPSeOz/81LZ7qpqmvVd3X5qgsM/Gi0Z4F2ej6jB4faxz5fvlVXXdjnJ6Z8cOVqsSeZcZfKovj8z+qJz6/Q+Ujus2drLxX6OgyC38sF53GhF42K4fkO9411pOhVTmOpUU5zmc+8c/0Rfvo6/srhDAWQvh0COHlEMJLIYR39XOdw+FwOL5/9PtF/asAPh9j/EshhAyAws0KN6dzuPDT94tz+XX65bLMVPoV4l9/AEiX5S/ehY/fq8rw1+HIGX2v6hz9clpmYlqe3FvSP5P8FZNo9ralLDog0ZLXlY/0rAYAMHxBfsZt36Pb2KbxGHtJ18Nf6528HvuVv62/oBm1md79ZxrDsmZSZE01JoyK6FZjr+pP2t276EvHGHuTiurjzWgVaU53dUWFFVmmMWGYzfRVV5/WZZaflF/Q1pgxP2NZbkw9lA7rb7UWv3d96g2GlmVBpgUBTSlV54zK6V0cvqCLnP2L8gs6WTcspSl5bNFOkdi7bkq3Z+0JOZBtY/Vj6z9d0mU2H7q5BXgz2rLndAwhjAD4IQA/DQAxxiYAZl0dDofDcUDoh/o4DmAdwG+FEL4dQvj1EILaRQohPBlCeDqE8HS7WtG1OBwOh+N1oZ+FOgXgcQD/Z4zxMQAVAL/IhWKMT8UYn4gxPpEqaDWAw+FwOF4f+uGoLwG4FGP8xv7xp2Es1NcjJoHmuOR6chtcRnNBvLOe3dS8E6sarB3hJPHElUVdhnfoLTlcIN6Yd6MBLb0bPmepHOQxc5KAln9xHwCtqACA9cflb212SxVBh7iw3XuMMhlZd3bL2NWvkapAKyPRGJPXtQu6zcwV5ncMnpSec3pPt4d55NqkrkfJpgxut7Csr2uMyePMnr4uS8qY+ozua31KluE9DQsx0YdawiAfU3TO2uPJ7Mp6WiPG86H5Mnba4H8Nrn37vt57Q+A9AmM4eH+kMq+fTyBHPd7TADRHb6mCshs0Xw3VFI91x1Ag5dZkPY1Jox5aT9TeyE22d3p+UccYVwBc3I/FAQB/BsD3el3ncDgcjtuDflUffxPAv91XfJwB8DMH1ySHw+FwXI++FuoY43MAnui30ght0dRm5Xe95RyQWyd5keVJRGgbMjI27yyzqDovr2PZH6AlhM1RXYblRpmy1vmxc0/acBZgxwxLSlQzZFvKLDPGLFPqPa5doqKURAvaocEyJRsTvaVdTFdZnm+ZXXnMUitA0yqWs4KiOoz2WDIypuas+899VU7ijZzREb6fJQ8kWM4kyT4cm4qX5dzbMOYLU5CtEV1PP96tTG1aqE8a9B05mjVHDCcUeqdZBgkAo6fkcWVB3z9NmgaLLmoN0wlDpluflvdPNnSbK0fIYWzHcq6Rx+wQZL1Pr/3txn9yOBwOxyDAF2qHw+EYcBxIrI/Q1WYym7eWece71P14kSWNeAZ8L97BB7TKgT3IAKCdo3gPlolMO/QNgx5J1XrTPimiXoqXjBgMi7puLmftxnPwF6ZZAEMNYFAfjcneVAPDMhN5zJg+AoCmYZIzFIVizBdNGRiKmz4+VyxPzY2H5ESzPAEtVRKDaS7La7c5yhfpMqUj1JFub3rCigfCXnXVWX1d2lDBMM1jvS91Uq9YcVaSNabYdBtLR5mO0PfieBvtcV2G56JJHdK7YY09x6KxPJSZqqvOyb+b6921+m78J4fD4XAMAnyhdjgcjgGHL9QOh8Mx4DgQjjrR0l50peNUyIqeR62xIlmxF1nDkAllKJJW0/KOo1P1cf2blV/rHfmsSfy3VSa3Lo/LR3R7mMstL/SWLQFA6ag8HjK4beUdl9FlullZxvIKrc5JPjFdVkXU3oPpOUp9tbjdpPKO01wmRyOz5osKVG/w6pa3YEz15hz7SbbA9x8+p8tUSbpq8aQdmsNWtDjFLRuR6ZpjPPbGmBFXyvsngC15rdMeBstCASBdpjjwxlxMbch30eLseQ/Fktf2k7QhQfPB8pjuJ6kH8+pqTwHak7WyKOf0zfZK/Iva4XA4Bhy+UDscDseA40CoDySATp4D83P6JX0ZmxQJQwfDMpiZb2uTeOOh3sHiFYViBaan+1uyNqZwrHtVF8jcM0ztQDKptiUjMwIKsdneyfT+7c1u6jI8rpZsi3/WzTIs0TKe89FPLYvjV39mTpVhk7QyawTmoTllBYBiFJYNasoIHrTxON3LsL/Zq409awEgT/n2SktWcCkae4tqIEqpOabnfTfZ23M0uymPLY8+lpNaXohcBtDz0zLlOWFHsGRsNPettHRM4VjPnmV9TGUCuv+WFJApRzPfK1VteUH2Q8PdCP5F7XA4HAMOX6gdDodjwHEg1EcMQJvNA2ZC8kZm46q0nTI7um420dcfNfIY0k6ylVGbU9tbHmt5MmXrc4YXVUn+1lkBhniHnrNnA9r8tbyfeGcZ0H21dvqVl6iR9ZvHqGZknuZ2W5mWOaO4tRu//OF5Wa/hqZkgGixT0m3mmL+WFySPfc2IGd0q6ut4h94K/pUmVQPPKUBnYW8P6UKca9FS9wxdksf9xDu2lCHqOuNTjdUrlhdxtw8PPkuRxbDmNKNieOSmKOCSRX2wKom9VAFN13QM5RDHZmdPXwu8Tln1MFXmQZkcDofjDoYv1A6HwzHg8IXa4XA4BhwHI8+Dlquw12Fo9+Zpu+nekfFyRhSvBvFnlkyIcxRaTFm7QGUMKZHKf2hUNPyKPLaCpdcoQlm7oLnM0UuaFI6p3hLCLnHAljcac3y5DT1oDeLdLE6WZXW5jd6c8NH/oDcjzv64dO3iXIyAHiNrTmWr8tjiDgsr6pTKB7l3rLcczeJked7nV3tLI1vD1phRbkxDCshz0fIMrM72TlDB/VDR42AkiACQpnKWByzvWVjvpvICNcrwPOf9AkD3IyZ6ew1X53vLHq13jHOcWhEhUzQXOfmDeyY6HA7HHQxfqB0Oh2PAcTCJAzpAZu/mudEsiVpzXNro3bTFWcjDzK5hbgYOkGJ4LZGZXDxv3Erl27PMVmkqWYF52rneQYjYdIsJfS9LesfjGAxag2VbltyqMSn7Ycnz2Gy1gtPo4OgWxSWPz/6YzuzAwXGGVnR7GpSTzwpmxDLRVNkaV30dB7eyzHimkBb/SOsML/+Q7KyV65ClkVY/ONmEFaiIqR8rFySb5Ja5zfeyvG2tQFbdTO+A/ywVbVk0As0riz7bozyk1lysTxH1YUhFWa7J6wJgyCUtnpSLGPWwJytLWc16r7Xrxn9yOBwOxyDAF2qHw+EYcPhC7XA4HAOOA4uex7zs0HJvd9IWJ4Y1LmEOuDZjEDtEZ1r8IrvXsqsvAGR26RpDcsMRsKzIZy1K1Gq5q7OLsBWhiyWFgJb8qKiF0JG8rMheaXKFbw0bHDXx6Nb+QH1GHreMsWfX2cphfa/x78kym48Y3CFxxPkV/d3BCVatuWBxgzyObSPkAXP2a49rMpcj/OXXVRE0ibtk93UAqB0iOdpe70D5PDcAzZOakjAaorGTesy2HjTmGblAZ3aMZ0b8u+X2z/3g5BiAEU1wpHdyECv6JN8rt2ZIPLdkReXDvUM5WNLVHEUubBdof8kIpXAN/kXtcDgcAw5fqB0Oh2PA0Rf1EUI4B6AEoAOgHWN84mblO/mI3UfIDjKkOgrqZ0P/jrC0y5L51RclPxKMe7N0x0wKwDI/I1g7N7E6qe2X9Lo0idtFXc8ueesljaD8rQldd2tENiDmDPuJBq09ou/PXqGZTW0nVo7Iui3pXXdcPvfEjqYDqvM01kbEsl2SX3UmjdBibdn3yj1aG5m7IG3t1qyuJ7mlXwP2nuwanqJNkqPFtC6TXZV1796nnw9Luaz52p2R+rPWqG4z15Pd1s+wTZSW5W3LErX1P2to30rGcyUPy5GXdRv3jsvxTxlj3x5hLaK+ffE0yR6HdBmmEiw6j+lEK4FI6S5ZUW5Nt7lJ732ybLw/MmikehZWRMJruBWO+v0xxo1bKO9wOByO2wCnPhwOh2PA0e8XdQTw/4arieP+7xjjUzcrnEh1MTwto3aXVuR2c35Kb0k3atKcSo5qU6XTkb8tzaR280uPy4g6yZe0O1jzbpnvvtvSpkqD1Bqpgjat4xUpz0hOa0nF0DOyX+Uf0hHMWyVp93SzRhD8YW22J3ZIHjKj7589TWbqMeP+O3Ic2ZQDgEC0SuacHvvEkqy7fVnbc/Eu+exbu4ar5pzsRz6r+x6elQ+o+1hJlenkZd+HJyqqTKkxrM5Vp+T9kgat0d2SfSsYc7pekXXnDun71zalNCUxZVA4OdmeWlfPj1RelqnUNB9QXJRRzMp7hvtiSva1kNNjXzNcI7M52e7SiYIqU5yS/S8HXSbJ75khd6qQ8iIW9HwtnJbPp/FQTZXpVuUSaKpHMnI8jHwMCHl5f0vA0eGAYCrR4o2Vcf0u1H8qxrgcQpgB8AchhJdjjF8S9wzhSQBPAkB6etSqw+FwOByvA31RHzHG5f3/rwH4PQBvN8o8FWN8Isb4RHJE/0o6HA6H4/Wh50IdQhgKIQxf+zeAPwfghYNumMPhcDiuoh/qYxbA74WrUrUUgN+JMX7+ZhfEGNBqyaozxBsnDD4m0M9GJqu5ulqVOM8pLR1qN0m6s6R52yT/RFnKO+JkOy39uxZYNlXTsqXtt0qOL23we6EmOfLUjObTWhVdd1yU49otaU64/ZAsEzpGP5ryXMIYV+ZkG8c1152oyzLhuMELkqwOWc3o5Qvy/uVNbaVlHpX7IM1Nzbemjsj7N5uGrK2jScfCiOxbZVvXnTskOenqji6TnJX96BrcMnP/xaIe170tyTdni/r5NNfkGKWPaj6826WxN2R2uXnJ9ZfXNdc9PFNW50pXJB9fOKTL1Go0PwxpZoGefeWcplJTNPaJV/Q+VPWofO+siHaaJ9ZFsE1tHtfrSdjgdUmXiWU59wK7hVqux/vouVDHGM8AeLRXOYfD4XAcDFye53A4HAOOAwnKFNsB9Q2SHJG0rGmY6IxKw5AONXv/tiTq7K1nyPyoTDBoDezJ4eF8gADQGTUyBTDI3G2v6X5xnjhLspao6jZ2uU2G+RSJVsmsaHO3MSv70dnTZYbPyXpKJ4zg8eRVZ5mJiWWSFC5oU79MNEKiZHgPkmQuvaMllp0h2Z7CN7QUr/GQIT+rkLlb13W3cqmeZTpE8XU3NI3A0rLSrp4fI8/L9pSf0HQRe0Z2DYqrsi3HPhR135nqYFoMABoNI5sATT2W0gIASM4ax/T7U6b+W4RAh+S0rVldDz+PhOEt2CnKcUxtGksiTfN2zojuRAHCYlXXwx6NKmmCQb9eg39ROxwOx4DDF2qHw+EYcBxMzsRWQP4SfeYPS3Mhb8Sq5Rxvab1pbKapZ4y9LI9Lx3Q3OedbftXI4UieiVa82E6N1C1GvzgGsFUPx6hOGrn9OCAVAGTInAqGqoBhxqOmAD6Z3d5xre/5Ta3oOP1XpNmc+64245vkoVV4TpdpkYeYFZ87cUHSQ8MXdJndIC9sjusyVmCgAtFD1Vk979JXZLs57yQAZDYlZcE5JQGg8BJ55Bq0QoPSSo58TY9Z+TDRLBVtohc25LxqFwwznuM4G/Ou3TRUOBQjul0xIiURxr5tqJSo6nRJj1ltRvbfivGu4nobS0d3t/e3Kuc2LJzXz4c9Gq0420MXKZ8oxcD3eNQOh8NxB8MXaofD4Rhw+ELtcDgcA46DkeclgdYoyVVYppTSHE5+TR4zRwzovH31Gc0L1idJnmfQtiyHM3MUcpICS4lHlTeNeFScKGDklMFjk2MVtw8AEhV9jrn27I5xf+pbx+BJmb+zoohxUPWS4bGW2ZJj3zVmGOf7qy7oZ8heZDFp7U2QpLGgx4f3OdJ7qojpsdYinnRouTf338kZ8k1SWRbP62+j+rTsW8pIGsEatZrBmee2iCM22sPvwvhLRtKGu0i6auUKNd6FDL2b+Vd1mfIRWaa0ZHDL1I9OxtrPktflNq13SpZJGAkZOLdh0siRwIkUrL2z9hAf6zm9dxclBzHudSP4F7XD4XAMOHyhdjgcjgHHgVAfCNokZ+mJlbp99psyGMzq27QXGVMLVs5Epiws2Qund+8aVAObV5b0rktKney2vtfupKynMa5/H5NNpop0PYUr2kzkvIUdw+EzUhtTZcNMHL358wKAdl6ac7VJw9OrjzFjMzG7oTvL9USDKsut9aad0hSXyMqJZ/aVzOaukdtASToNdqZDJnB7yPBwLPYxZoaTLqMxzqa+0R6iwSrzeuzZtGc5JaDpAADIr8lyrSGLCiLZZdHIA0peh7lNVQSpWm8aIVLO08yuLtOhceUxBDQNmarpMiy7tOg0pm2tNfBG8C9qh8PhGHD4Qu1wOBwDjoOhPgBEijPbzd38GADO/aikOuzdVdohr2kTo0GmmuV1WKXU7a1h3Z5OQZqtiWZvLy7TRO2wia7vpfphmNEtQwXTHJNttDyiuK6O4UHHqoLcFd3Xbkr+rtendTVJ6kf5hJHnkTzmrHjQkdUs6/qbonS3lB7kr+jpXLpLlhl9WZfZO2EFOJL3L57T1+0+KK9LGMn0wqiUTCTO6AkSiQ6oHNbPx1a9SKQoaJc1F/m9rBrvIc8Fi15sG8HI8mvyue48bJQhj2UY8ajTRM3Vp3QTmRqrGdQU96NlUGOKojWUIaz22rnPiCdPVEfXCAQ3SQqb+qgcr6ShrnmtDTf+k8PhcDgGAb5QOxwOx4DDF2qHw+EYcBwMR52KwChxkxR8PFiynKYkmljGBWg5XMwZLlLEeXYKRsB9CtbOUeiuNlIelo5r3onbM3RR34s5rvak0eZNTlKgi9QNaRlzfNGomuVvtTnNGyevyLHn6HUA0CF5nuWJpys2kgsQf5db1WNfX6JEEx3LxZE4v0PGoNH47BpJAqxn3zos9V57d1vRDGl/wAgW36Lg+aX7e3P2cbJ3Tr6YMZ4P7wMVjPGgR5beMNo8I9uYXDeSBGT0u7DzEG906PvXSU7K7yoAVI/L/hfOaM1pa0TWzUH5AaAxTfc3hiNJCUTaI7pQoi2fD+9dAUZyBeMTePk98iR7YPJa0qM6h8PhcAwSfKF2OByOAccByfMikmQWdnbldz1nrQeAFAc/2dNmUX2OJFFVQ0Y2Iu3/RMkoQ9Ily0to4tuykZtv1WZRdkNeyAF2ACBdkvWwtxqgzancJd2gypK+f3pb1t08ZNAal8lsNvJDMq0xfEbff+8BWWb0BW2r7T4s72/lZ2zOyzIsuQSA9Kq8rj1n5F7coTk1pMdn4ltyim+9S9fTnDRsYkoIEQ1zN00JB1qHDX0V5d2E0UYlCds03EtZMlczniHlH8xdNhIQHJWUDsspAaCzS/0yqDorhyVTWsHIuxnz1H/Dgy9BY1YzKK0k0UWNCYOyILmk5S3YoYBpVu5Unh+pXWM9IYllMKSavMZwwDIreNw1+Be1w+FwDDh8oXY4HI4Bx8FQH52A9p403zKUmywaOQE5WBAHEwKAzKa8LmmYGLWktDGGLukyFYrctPBFTRmc/4isJ7XXO4rKxIv63NaD0ixK7+h6uB9Nw1Ny5BWDjriHd7Z1Xwtr0rxrjOuBbY3KMlYc6SRRSJyjDwBCg56rYc7lzsm5wV5mANAaJ4pryzCjSflgmfqlo5TLzqjH8pLlnJXswXa1AXwzPWgcAzlhKCgylNvPijXN74+ZX4+ooOaEpmsSG7KMFbipsEIeqIa3nvXe5U+SumhW1x2Iqhs7qdu4dT+NWVuvFeMvy+s2HjMCnVEbU4anc5m8hPPL+h1rkAeu5Q2dozVmz1CIcVAmDuBmKb1e+9uN/+RwOByOQYAv1A6HwzHg6HuhDiEkQwjfDiH854NskMPhcDgkboWj/tsAXgJgxHCTSDQDCudl1fl1ito1a0hlKPi2mTeQ+LL6tOaCkiRd4kQCgOaZlt+rucNkXR5bedk40Pj2/b3vlaroMg1KLtA16PCaIf3LrrHXlP7t3VsibszIq5gh/rmlczYgTTypFYi9wfkqjU8B5kVj0njO5DE29ooxXzKyTGXBCPpO90oa+Qgz27qRvaRUABDodqmSroe9zywZKOf9tHImcmB8M0ojNchqc574Z84ZCAAdehWKFw2ZrOEly8kVrGD+nItz6wFdd4pyg6aqup7aNI11NHh9mueNcV0P77sMXzT2S0Z6R7Zs0vvCcwPQz54TXVjv/DX09UUdQlgE8BEAv95PeYfD4XDcPvRLffwLAL8AwPj9vYoQwpMhhKdDCE+3q8Yno8PhcDheF3pSHyGEHwWwFmN8JoTwvhuVizE+BeApAMguHo6cZ40D+FimwdR35MnSoiE1W5W/FdtWKnnKQdcq9k4lP3JGt6c+Ja9rFXUZNlOtYEYt8rwz8zzSKTaZATvhQPESBSaa7J3XkRMrAMDwOSpjmInKU7SPn3mLdrpZgPRrSBF9tXfMkNCxaW1IATO79HwMSseSB3Juv1TFyC0oU3yiazgU8nywvOPYlDZzc1LdFh3AprT1jvG5dkGX6dU+QHviAUCDnjXnZwT0PE9WjHd8RdZdXjLGrA9kd2Q9zVFdz5Hfl5Nx5V06A0F6Tx5b77iia4x+8XgkDGroRujni/pPAfgLIYRzAD4F4AMhhH/T/y0cDofD8f2g50IdY/w7McbFGONRAB8D8IUY41898JY5HA6HA8BBeSYmdGxc9qSydptLh+XvhpW6vZMlVYER75jzwnWM2L1IcBkjyA2Zd5aCgb2NWL0BQJnkMWHsdBOFkts04jinLTqEjw1lCNEoKSP6C3vwWZ5virKwTGI2/63db7o9U1VWma7xDHkX3VJLMNXB3mAA0BjX16VJXWSNR32K6BHj/kxzWd62HLSLTXYAKB2VxxalpCi1mi7Dc9h6f4Yuy+O948a9rPlB6iZWTQFaiZIwaLDde7mM9S5wGUORRe+GRXFd+mHyoDbmR4cUNlabeX5YsaV5rDnevrW+XMMtLdQxxi8C+OKtXONwOByO7w/umehwOBwDDl+oHQ6HY8BxMBx1iIiUU61wWt7Kkn+xF5kld2oSV2lJXNirzaonsyPL5LY0f1Wbk8dWFD6Vw9HgUtljzeLnc2ssSdJlLP6suCwrq00bfCvJyJjXBrRXWW7DiEpIUd046hsADF2Q56w8k0169hb/m7BkbATmHMdf1mO/+Yispzqv67ECtjPfnDW8MLVXrP7uya/T/Q1pZGFFlqkYslR+hrVZPa4s6+saUe/4utyabnN1jvZmDB47s6Gv49ygzOEDek/HkhlWjlI+0w3tsme9Q6oM3d7igDuU7CC/ou/FcsGd+3Q9vH/TMLyIRyi6oHo3PHGAw+Fw3LnwhdrhcDgGHAdDfXQDklX5G1A9RFImI/h2P55E7OVnBTCPFOTdzL04Jc0OpkIAIJKEzwqEky6RiWOmpKekAIb52yUPS0tSOHpOV763JE21ZF1fV17qPfZKXmQEiOHg+S0jPJfyQDUSGbAJagbBpzIcjMu6rnTE8FIdpnx3RsIK9l4EtPzM8rIbupjsWYbnjOUt2Kbrgk5RqMbMeoZM4Yye0fxAfYYCNxkehh2iCrNbesws7zymXqx+MA1pefuy96b17FWOQoMK4ftb3r71SdYr6np2T8hjK88kyw6tvpfIu5bpPWtuvFb2xn9yOBwOxyDAF2qHw+EYcPhC7XA4HAOOA+GoE20gt04JMokThhFdivkhi+fJkiSsPWRwbLk+eOwM8+H6XsVzJK2as+7FcqPeruhWZC3mf1nqBADNIeN3lZpkydqak1xZb7kTy/UA7aZr9X7kocYAACAASURBVENJoCzpGyUxNnlSOjf9bV1m634KOWC473OS2mAkc+0nEh0nvgCA8hGuxxj78T72Qtgd2gq4T+EUrAh7vPexk9X3SlZp36Wg+8WcsOWuzjw2oPnm/FpviaeZRJkiJ46cM1zql3rv6TQmaK0wQhUwb225fnMb00aSXB5HTtAAAA1Drtgv/Iva4XA4Bhy+UDscDseA40Coj9ABMrtsYpHkxjDty+SRZeXkq0/J43RZm1cskzLNO4MOUfeaoRPGJa3h3t56LPWyPOFAEfVaBa03qiz0lklZNE+iKa/rGHVz5DUr0lhulTw+DTOxsCqv2zUC9Q8tyzKWJx5Hudt4WNfDMsxuSrdZUTqh9/MBtHlrSUU7WVk501eAftbsYQgATZY5GvOD+2ZRXOz1Z0nNwFLArpGfkeRnlnzSonnaQzQeu3q+8vPIr+h6KotESxqJP3Ib5D1pPNfmmDqlwJLg6pwuUySPZM6PCGgqKG16Qd48J6y5Llwre+M/ORwOh2MQ4Au1w+FwDDgOhProZoDy4ZuXsQKkMEWR3dFlCqvyuHRMl2HTzTJJE2RncO5BAGgRXVM+qu/VyZO5V9aKCg7YYtE+TNdwUHxA53kEdBBza/c7VeLdb1035wRkM+3qdZwH01BCUHIDi1bgvI75NaMemj+WyqG6IMfe8k7rEoWTNYIQKeoBmp7JlHXljQlZlxVQnhUDlncrj6vlQccKm6bhGcj9n3lGD/7mA7JBlmqKz1nUx9Cy8TxSvT3trHmtytAjshRIo2dlo/aO6KWM72WNa2WeqEMjnymrvSxPZzXPjbnICU3A93LPRIfD4bhz4Qu1w+FwDDh8oXY4HI4Bx8HJ84jHqc4Rn2jxMXSuPmVIh8hrK3R0Rc0JSRhljcDjHAydpYEAkNkz2tgLxk8fJ+m1PL0YFi9o8YmFZfLOmzC880iylzSSnrJ34OgpXWbz7ZJcTxh8PPOJHIEQ0LKp9pDh4Uh8ntUvDnpfW9SDll2XbbS8yiyOmiPxtQv6wbKHpcXZc8/M4PU09pYsNbcuj/dO6DIsOS0t6ok2fkqO0erbDAkdDaMlxSsfNvpKp1pDqoiai1ZiCfaMtGSgpXnZNytJAfP6VrLdBHkWc/ISAKjT2jV6Uo9ZKU97Xpd1v7oZORfrM1TmJp/N/kXtcDgcAw5fqB0Oh2PAcSDUR0wYAdP78M5LksljeXEVVqW5sJM3gtxQHrSCESCldJTubdABbE6xiQoAE8/Junfv6W0SZjcNL6pRkkQljAHKGoHgqY1pI1BSY1oOPid1AHQg+obl1UU0Rt6QutUpMJIVhIj5AIvmYY9LS9alZGRGgHkOuGRJ+FrDhpma622WBsqu0C72TuTXyfZOgNAyqKChPVm3RYNxI+tG/kweyPHv6Xo2npD3SlUM6tAYD+U9acgV2RMybSRy4MBZdYP2SpMMlJMNANpruHbIoAVp3lvSVZbalY0EFS169o2x3u25FfgXtcPhcAw4fKF2OByOAcfB5EwMetc+co4zYydZmaBd/TvCMYBjWtvNmTXaEZ7Q98pu8b17m0WY0Kbt5tvkuWRFtzlJdIS1Q80mOatSACBR03V3yRMxtW7khWMPPsPcbJMHn5X/L1SlCWx5x7FCwBpXjlPMqhgA6Az1HtfuKKlQdvV05tyPlmdgd8jgXjivY9MYs5HebVQBsCxGa5NjtxuxlUkBFXO6zYktfj56vrZJncDjDAAxI8+1Rgx60QiAxYoOzo8IAB2qW1FMANI7sh8cgx7Q3r2WwkSpRaxgV1TGeu+KZ2V72CMWADLbcoxKS8b7yzkcNyhWuuGx/Nq1N/7T/sUh5EII3wwhfCeE8GII4e/3usbhcDgctw/9fFE3AHwgxlgOIaQBfCWE8LkY49cPuG0Oh8PhQB8LdYwxArjmJpDe/+/1b186HA6H45bQF0cdQkgCeAbA3QB+Lcb4DaPMkwCeBIDkxBhaozfn7ywvLvaOa8xqHi61J+uJhiSLo/BZnk0subE8JS1PJgZLwoYuGVLAY7IfKUuSRPe35GiW11R2i8bV4AWZX7VkhuzhOHZGj/3ye5mD7M1RWzIyTv5g1RMoqLoVATG5I6evFUyf5wLL/gAgtHpLCDObBkdNfbN4W45KaMna+FzOkG/WDpFkbtvg4+m5soef1UZr3yNQm613w5I5jr0sj7cf1GU4WH63oMuwzDFZ1y8D731YY9Ygzz/OnwkAOYqoV5/U7WFPYksGyhEyG8aczpJUNUNJC6yomq/97cZ/+hPEGDsxxrcAWATw9hDCQ0aZp2KMT8QYn0gWLTGiw+FwOF4PbkmeF2PcAfBFAB88kNY4HA6HQ6En9RFCmAbQijHuhBDyAH4YwC/f/KKISF50CcqfZsrhJmTEpfautuM7hyUf0a3qLiRL0lTqGBIgbp9lStbmyfwvatsk1uS9ajO6X5lD0j2uuabtPWU2G6Zl6BjmNx13DC+7wkXZxspR3Y/KUQpSdY+uJ7C0yqAMan1558lxzVzWz7k7JW3JWNHPJ7NKcrQxQ+Y3IfvKcwMAQsOS3sk2NpcaqkwgT824bczXcRprY3g6Odmm7p5uT2JBzqHWjtZPFs7LMarepV0DE3uyTHdC82nZS7IfjUVdTzBkbBvvl8fDz+mI/5W3SRfgsKnHbOoZ8va92wjsNSPHtWl4HbKc1JIiNoh3ag9ryq9BGk+LqmsRiRAn9ZjVSa9YuVv2ofNfb7z11w9HPQfgt/d56gSAfx9j/M99XOdwOByO24B+VB/PA3jsDWiLw+FwOAwcTDzqZERuTFIUjZo09/MLOjDwSEFes1KaUGXmpmSw3lpLd6E+KWUe0YgAVdsj0/GIjso0OiTbs7OuN0lzU/K6zrgRUCcvzeZGSrvH5aekaZtNa3qiVNHmbmdDmpe52YoqUxuR45E2vDlbZWmWjUzpeoo52Y/lZf180kVp8mWyuh8TQ7KvO+O6X63LMkh0clxTD7W87EciZfAKbVJdzOj2dFqaDknnJSUwVND3r9TkmLXy+v6H5rbFcTapx35lZ1jW09ZudqN0/7rR19ao7MfCuI5qtrI5Ko5zGT0e9Tk5h+cXtlQZC5t7st3lJS23GhuR82rboM/W3yn7EfJ6zFSc744V5U0eji7qQN+lsnwXh4e01KsyLOfnxLh+N9pES84V9Hpyrjkjjgvjsow5f6/97YZ/cTgcDsdAwBdqh8PhGHD4Qu1wOBwDjgPhqJOJLorEyyYo4lQuo2VBSXKBmlowkscRrPj6QzniSQ1eEEXJkyYM96sWRe8bn9GcH3NceYPLbHeIczPkYMxJTxU1D5ZO6X6sVyQPeHRS84ktkhdtVrQ8cKcppwJzboAxjoaHVmdHjsfEg6uqzFhOcnPZlMHH5yXfOW3xrZfHxXF+VEui0tTmYDznssH9H5/dEMeNjn5VWvRcUdT339yV/Tg8taPKdCmYfuGYTtaZSsr3Z4LmLwA02rKNqYQhS6VTjVU9Fwpzcv8obdRTbxuekdSP5JxuI2N2Vo/H+rbk7Dtlw7WYpJEhrduYPyTnWSGrnw/vl6QMl8vLtA5MFvS7uduQc2g6r/fgmoflfNnYlXte1l7aNfgXtcPhcAw4fKF2OByOAcfBJA6ANjGr29IkTk1pEyNFpsnIkDYfTi7PiuOCIaep16WpFAyLYnFSmlwru8OqDFMoFtrkGVkY021mUz95TPe9Q+ZVOmHQNQbGviP72jmuf3vnCpJCYhMZAIqzsq/5lKamXl2elicyuh9j8/JeTaYHAHTJxKu2DNOWGIqcQY9MzEqKYHpIm6Svrsg253K6X62qvj/TPBY1lif6juWLFqx5xm2aMKRdlaaUAlabRpuJGjt/dlqVYcwe31Tn+PmsGm22cHxG1rW8N6LK7OxIKuj4/IYqs039SI/rd6FDssuukWSE6dWaMWZ3jcr77zQ1FdSme61XjCwFhHJLe2UyhdTckWVMieE+/Iva4XA4Bhy+UDscDseA48CoD0ZmWJrW1g4se2jdM7uuynSIalic18qQ1aTcTd05rT3o9ogy4V11ABjNyTLnVnWw2qEJaaYyzQFo1cVoXtM1y1vSY4ypEADY3DbCx75T3i9d116Po1nynjR2l4cz0mznXWwAWJiRdJFFazBFYZnNqsyWNpGHJyWNsVfXpuQQmbaFlJ5TXTJb6zUdBOjwojb/V8qy3cNZTWuwWmVtRz8f9nSbNNQarC7arupn2GBVjuGRmyC6pjCl78U0YKmmx3V8SM4XS6Flvb+sWmAVFwAUh+V4bJQ1jXBoTCt8GG0asysbo6rMXknSGElDNVXvSDrEogWHSclVbWgK5d7pNXG8XNbtYSwdk+vbluEleg3+Re1wOBwDDl+oHQ6HY8DhC7XD4XAMOA6Eo243k1i/IL3GkiOS5ypbnGO+t7wJFGHq8q7mgu6flt5wLxq80+YWeQW1jWDtxLEFy9OLaDiL22W+2fJA6pAn4JWSIRcsam67vCN5uJ2S5jdfacmoXeyNBWgvtmnD+4p5QQssh2vt6ue8R/zmouGtx+D9CwBIknea6YlHiR0Wj+l9j2JGz7vd2pisx3hm+TTJ6kb0uE7ROO4YewiW9I/Rpflx7JCWtV3ZlVx/xpA08phZfCu3pmHw4Utj2+rcblPOfeu6IxPyupMvLKoyo/dRchBj7CsNuddgvb+TM3L/am1D74Uwl8z1Arqvu1n9jpda8tx2Scv8WBI8mZNzI3UTSa5/UTscDseAwxdqh8PhGHAcCPWRSHcxPCclNtWqNIEPjWgJTpm8rywzMVOQ5qYVYJ9NpSFDSjRJ5v+l9XFVhr0ro2H6D+cpH2JbS9Z2yQyyvOM48MzwhA7Mc2VFtzFFgfknR62g5pQ7zujHWkVSQaYnYE72tW4EKlqakUGh1vJasnbfpJQyfWdlXpVhiVjKCpRPY9002nPoiGyPRXNYYK9Di6pjeZ5FvXCQH0vCN5SX8zNp1MP5GdlsBoDL22TGG9K7dlOOWa6g3w324EsZsrYLu2PqHL+LjYu6r41R+d6PLml5LQfSWi3pemon5f2zx7RH8G5Frh8Z473bLEl54F3TmlJiSsJ6f1g+2tzTY1+lBB5DaUpwotIh/An8i9rhcDgGHL5QOxwOx4DDF2qHw+EYcBwIR51KdjFNge9ZFGW5anLiT8tteIRccpnPAoA9kglZweJHsrKeTFbzV7u7kluemdK88eFhdqvW/WKusFbVEqD5I9KNuZjW3OF9S1fUuSsUoWxjW8vYxom3tuRgexQ8f/OSdu3NP3pJHK+VNXd4eEyOx8Ko5iCXK5JL5ciKAFCg51FZ13Knw0cln2jxpswnTua1hG6zpuve4oD/01qOxuN43tjnYN56xoiuuE1c6vSw5p95DtXbWlZXv0yS02G9z3DP0oo4vrCl29z4mgyVkH2XdrG3UGC54oneSXGPjeu6V6tyDlt7TGVKJFw03NwTNPYcXRDQ74sVhmC3IZ8Prx2AXoeS8/odG6XreA20ZIjX4F/UDofDMeDwhdrhcDgGHAdCfTRraZz93pw4N3ZUmsRWnjwOIF8zvIQmySy0zJCXTku51/yiNsHYxLFkfo/MLYvjSyVtWnOAcMszkaMA5pLaTGO6Zqmo2/zi1pw6Nzci6Zh1IwrgItEzGzUjCh9hwch1yObdTFGb8bM5Kb/64zN3qzL3zEl5HgyvshPjcswqdT0XuD0Fw/xdI9M2M6Gfs+WxNjctKRvLLD1zTnp8vu+hV1SZC2VJLdw7psf1XFpSDWdWp1SZIlF+y2Xd5qUHJDVWbmjqkKmOpDFfKktyjFpVXc8jC8vq3DMvHBfHh49rL1DGlYruxzhFoKy29LOfOyHrtigtRS0Y8rdxmosrRns4iQZLWQEdXZGpREDPV5Z8qhyc18G/qB0Oh2PA4Qu1w+FwDDhC5KhCXCCEwwA+AeAQgC6Ap2KMv3qza+5/JBt/67PSTD/VPCSO0+HGQbKv4V+de78694/v/ow4XunooEzH01IN8Gz9iCpT6sid3IdzF1WZr1ZOiOMfHn5Blal2pfkyndQ79l+s3iOOx5LaTDuUkvTE6easKvPO/Bl1bqcrTayhoHetf2frneL4T4+8rMrkgjTvXmlob8F3F06J482OVoYMJaQJ+K3acVXmcFrSOsMJnWyB7/+OwquqTIn6PmbU87nSI+L4scI5VYb7DgCnGnK+Pp7X1613JK1i1fNs7ag4nk5pj9xDKUmz8L0B4L2Fk+J4rdObvvp3G+9Q5z488XzP6840ZWAtfu4A0DG+8ZKQNMovnPxLqsw/PPEfxPFOVytuNtuyb3dl1lQZvm4iqWm4r9H7e29Oq6b4mVW6mubh+blljP10SlKQ360fVmWOZuS6VAjyXflbHz2Lk9+tmdKPfjjqNoD/Ocb4bAhhGMAzIYQ/iDF+r49rHQ6Hw/F9oif1EWO8EmN8dv/fJQAvAVg46IY5HA6H4ypuiaMOIRwF8BiAbxh/ezKE8HQI4emdrRvHVXU4HA7HraFveV4IoQjgMwD+xxijctGLMT4F4CkAeOLRXHx7Vkrt6lF6IL0rqxfzky3Jr/7Gvf9GlZlNyiYnWtpj7JGM5C6/WtVeXD8zKqVUqx3Nmf/YyLfF8fG0ruePavLcgxntZfdzL71dHP/hw59UZS51JFdmcc33prV856mdJXH88VHNJ85kJC/6oYLmSZ9vyufxw0MvqTLzKUmf/cLpD6kyn73v0+L4k5uaa39yTvKtL7Q0Lfenh+TzWUrpvZRP7Mqg81bfn0nLqfrn8noP4cWmfvbH6f5jCf1Nc46kme8p6Lm42ZG85IcLWp73KvX/uCHNnE9SpEDofY7jJDm9OKp5/R8dku/hd/Q0wztyUno3nNDz7nxbP7PjtJr8vbv/iyrz3pwc65dbWsL3npwcIy0gBNY78jkupXR7RoLcU7o/o5/hakfyxKNGX1tRtqDU1c95mtalw6kXVZnFpFwrdrty8PM32bfr64s6hJDG1UX638YY/59+rnE4HA7H7UHPhTqEEAD8BoCXYoz/7OCb5HA4HI7r0Y887z0Avgzgu/gTK+R/jTH+1xtd89ijmfjHn5MmbzWSV1tSS7vWyJwZT2jvntWOlMqMJTR7w6ZKJWrjaTYpKYpLbS3tOpKSEqDLHW1uDgf5W1c37sWirdmklgCxCfrWjDbBTra0F+Z0Uj6/1Y7+7Z0m77NC0HV/oTYhjj9S0HKnbzXkvY6ndXtWyLtq3gialSQPsW80dGCg9+fl/Z9u6DafSMtndqal58vd1MZ00OPzH8taSvVTw1IS9nUj38CjGfnQeAwB4L05SX20VEZC4AtVSeH8RFEHr/8SeWa+I6spnD+uS8/Zt2V1wKN0H2P/wYLs7Nfr+hk+ktHn/j/KM/mBvKZweB1YN+brfWn5fny2qr0F/3yB5HBNLY28Oy3HmtcFANjqynNHU1ou+OW6XGPel9f1nG3J+Tqb1N6UjSipDb73j39kA9993uAB0QdHHWP8CnCT1AMOh8PhOFC4Z6LD4XAMOA4kKFO5m8RX6tJjcCkld0qX29qWTJBJvtzWu6CnW9Jj7YHMiipzpi1N0MPk9QcAyx1pdnSjVmtUI3nZ1bWX3bvzZ8Xxxbb2lGRvxVOGiZ4kk/h0WyszPrP3VnXup0afFseVqB9pmnaXP19bVGXYa+rlln4+nys9IY4/NvotVaYe5TOsRv0MP08eY2/JnVdlmApiz1YAOJSUnpprHR2LezYp6ZGv1nXf78tqj7WTLWlKf6XyqCpzOCVVQZOGd1yJzO1fWf0zqsxfnfyqOGYFDqA9VY+nTqoyPIdKXU2zfHJXzqEPDWtPxRfp/p/Zebcqszj1ZXWOPW5LXf3sf3n9feL4Zye/oso8Ryqcyy1Nz2x3pTKkHvU7tdWVz/6XrvyIKvPXZ74gjpneA4AVeqfXOpdUmVOqjVoZ8hvb7xLHf2H0WXHcvsl3s39ROxwOx4DDF2qHw+EYcPhC7XA4HAOOnvK814PCibl4zz//uDj3Zw9LT69SW3NKjC+eN4LOT1NA+baWwVzelpzSwrjO2/fgmOQlv7WuI+wNZyRPy/nvAGCPEgUkjTIceNxKHHBqU0YsOzGpPbZeWZ9R596+cEEcL+Q0H7/VklLIVNAc6E5LcvTPrepwLhz4fGlU83DHyPPtcl0nW5jISC5zz+Ds/9vzMuLgo/drHnuSJGrrDR3VbLEgx8Pq+0pdy7/qHelFtl7VctJ3zMg2LWT1eFysy/2SbELztldoP+fk9rQqw5gwAuU/MSHnwgu7OgLivSOS2620tVT0+S15XcdImvDO6XPqHEfUSxtjzeesKHw1GvsLFS17fGBEvr/jaS1XPFmhiJ0J3Z4E8fpfW1lSZR6fuSyOpzN6/6jRlXtDZyuTqgzPRX7nPvfT/xGbL22YCjv/onY4HI4Bhy/UDofDMeA4EOojf/d8PPZPnhTnxgpSKlMx8iFyWvhDQyr2k8qxVjXqOTwmTQwrx1mLPKJGclqOtkc5zRZGdHus9PKMb7xwlzh+24M6AcCFPSnv4dTyALBS0vKzuyakrG6rrk3082ckZXLPCZ3vbrsuzbCZIS01q1DuukTQcyeblKb9+S0trZoZkXXXWjrYFZv2Vi7KlZdkv976NiMo0zelFDC1oCmDiRFtNvP4j2T083j2gvRoVLkgAbxK+Q8XJjUNd3FNmvbjo7o9Q+QFeXFNj+vhGUm9jGW1ty1jpaLnFD/7s1uaeuB3FQBylAe13tZS0Xxa0n4LQ3o8nluRtNuRcU0p8Xzd2Nb9GC7K/lt5UUeI3jz1gpZvxpykTIoz+vmM0/rGNCEApIKkRReH5Dr12f/hs9hw6sPhcDjuTPhC7XA4HAOOA/FMTCW6mCpK86Ddlb8JBcN02q1J89YyracpnrCVkJ7v1enq36PtLUmHjC1o0zZPphKb/oBWcLy0ruMvh7w0g771yjFV5p6j0sOy1tZ0wLBBz3Rj79/a4qw0ZSdz2nRjU7Lc1GoANmWD8Xy4zNiQNr/Z1E8YZuL6pjRlJyc0FTNxQgb92WtqeuTIw1IdsFHW1JBlNk8vyDEaMiiuXE6rdxhJCprFcxwAuhTb2RrX7ap8PoUhPRcaFBDruxe06qPblGUyRd2vjZckXZM/rik/i7rkczPD+pmxAupyRXvyDuflu9jq6oBc9aZ8P47O6gBUWzRmzbauZw9yns+c0AGxmNKxwGqvrqGU2W7I9jA90oq6fdfgX9QOh8Mx4PCF2uFwOAYcvlA7HA7HgONAOOouguIqGy15bHneZYizOX9Fe/dMLEl5VSGteULmtk+MG/ci3snilIbSkr+7vKv5NOZ7m009pIuzWl7EYI5rdUfzpouT2uuwS6HCV4zrxkimxNcAWv61tmdIGqlvDy1omd+lkvRErNQ1l7k4Lcfj4qqWmiXTci6Ua5ozZ49TS0J3blfy4UWD52dpFQBcKUkZaDGlrxul6165rPcn5qaojYbscqYoudxGR8+hTlc+s50dzbWz5+zxec23XtiQYz0yZOzNPCQ977YqOpg+y20BvTdkgT15N0q6H6O0r9EwZH5zJJW1pKv30Bqz09ARMl/9tpRYpo/o/ZtugZJPGHsqzIeXtvWYzc7KufDiGSlDrDf0vtQ1+Be1w+FwDDh8oXY4HI4Bx4FQH4CWGFlyPAZ7tZ1Y0J5ebZKjnb6oAxWNT0rTbXpay4S2yOS6a1qbiZs1ab5YAZcyFOhlfFh7vi1vSsrk8LSmQqok/Svk9HixZxOgzc2ukYOO5UWWjI1lY5YXVyDGpNLSdMTOnhyzjtGeCxV5nSW9Y1O/1dHSpVdPy6A7D997UZVhtI16+sGFkqZn1rYkPZLJ6jFj79qNmqaUeN6zhA0A8kV5bndPUwazw3LeR4POK5IZ3zaeTyclr+t2jXoymgrapMBVVhCkvbqcZ1PDmmookUfwyoYes4fvlsH7rfnKAZcsSgmHZD8mjfawWLKY1u/m8hkpaZw4rGnK4ay8V21C3mstqd/va/AvaofD4Rhw+ELtcDgcA44DoT7SiQ6mC/Kznr36ON4voE21jareOU0mpCEyOm7s0hIdsF7XptOhMTJJjXjDHLjJMq9e2ZLUy0hO76IPzUgPOjZ1Ab0bzrvagK06YRWDpV7hHel2Tv8+Z1LSTN3Y0WMWO7LucsEIrFWQ7bG8wQpZacanjF30zbPSlIxZbRYePSapsZWy3vkvVeW4DheMYFdntbooOSbNW8tLtl2Xr0+7ofvKz8N6Ppd25HPloFUAMEqKllkjuNOVPUnFLBnBjIaJgrSUGjVSaNUr+jmfooBUADBzv1RZWIG0WL0zW9SxnS9dkUqdZF6/L9zurDGHOM75lW0dd3xqXN4/a3ghVpqy/5aHcnZKKlWahlKl1JB9b9I4W1TVNfgXtcPhcAw4fKF2OByOAYcv1A6HwzHgOBCOOiCqqHLsXWTlrntlQ/K9uayWKaWpXku2xV5TFv/MkrWdsvZaYq8tyyNp+YLkN/dGNS83PyH5RCsyHnNuHM0OAPYMb6fsjOzHqCEPLFHfrCh8JeIO83ktQWJvTiu5QY2imqUMyRHzlHlDujlzl4yGtrqscy9yVMTxnPaWY49Y5qwBmJ8rI+TNafGH6RE51pZ88/lL0vvsgYUVVebyS9Kjce+uPqLyGZw5c/3ntrWkkDFheBjWKApeJq/bc/fjV9S5752fE8dH5rZUmWNT8rlaURqPLZJHoRFx8KXTMjJgYVz3Yy3KfZZ0Wr+/PF8tsMfnpuFN2diS71iYNNpDewhzs1LCd9lYX66h5xd1COE3QwhrIYQXepV1OBwOx+1HP9THvwbwwQNuh8PhcDhugJ7UR4zxSyGEo7dSaScmsEsBUNhTsdrWEpcjU1JOtLyj5TScc61c16YTe02x6QJoic3ChJY7rexKuRfLygAgUZbUS3ZGl7mwIuVGY2NaUshBdg4bgZwyBV03m+Rbq3rMZuel6aY+FQAAERxJREFUiWVJxHIZWTfLIAFtJlqSNaYa2oY8j70VG1c0rTF0WMqmDh/WgeEvr8vr3nr0giqzRd6lVuCbsTkthdxel89+fkGb8Zyvc8KgnSokYWwaVN30PZT3cleb1lMFWffpNZ3H8NghWc95owx7T1p5DY9MyLl3bkPXY3nJguYVJzIAgLGsNO93DXngBAU6s+bZVls+e4tiY2/onbKmDifIE3HT8Ph8ZF4GH1vf1dLV/JR8PjyGALCZ13X3i9u2mRhCeDKE8HQI4enWbu+kmg6Hw+HoD7dtoY4xPhVjfCLG+ER6VG+EORwOh+P14cCCMjFy5I1nKR8Y907roExnt6XKgvOrAcDJCzJYz6Kx+3zxzLQ4ftvDp1UZNrnOrWoPtu6QNOUsmiVXkCaYtdM+RNQDe0MBwL2H9HiwusbyrOI2WeqEpRFpqlnU1HdPL4rjVcNT89ikHGsrEA5TWoXjhlKlKiktDm4E6DyC2w1t2nLQodaG/ogYPaHrrg3L+blQ1NTYZkqaskyzAMA05Q5l70FAexBy8B5AP7MHFrXqokB5HUujmhbkui168QjNhVTqxsGCrkdxjFQwljcnBVXjHKiAfo4cFx4Ajt61Ko6tvKhMMzG9B+gY+MMFPfYMVnEBmoIspnU9r6zLdenuI/J9tiie1/7Ws1UOh8Ph+IGiH3neJwF8DcC9IYRLIYSPH3yzHA6Hw3EN/ag+fvKNaIjD4XA4bLxhHPUuBavnXIMA8Oy37xLHjz+meWOWllneaNl5yTtZeeq6x6SUyQqmz96CHGkLANojkgezcvux591GWct02Juy/K0pVWb2w5fVuQpxyVZfOZffg4c1v7lalXI0K8Lf0Bi10ejrYeI3Lc8zllLNDWuOuJiV17HcE9C5DtdKWja1MCr5xPa8NiKt/JAs17TmB3On56wcnwuSt7W4VJ7TVo7AapScOXvWAnq/YmtPc+aVLCWoMCSnp2kfaKqo31V+nwHN084U9Puy25TP0UouwNxyJqFlfnt1jsKnIw5eomiTBYOjZozl9XrCz2fdmGd3T8r15EpFc/9p8vDk9SUauUyvwTlqh8PhGHD4Qu1wOBwDjgOhPlqdpAqGrjzojBT06VlpdlwqaY+1RlM2mXMNAtqksAJAVcirbHlFB7CJTVlPeljLhJYoKYAVBH/lInl2GRZOZVi2J/2IlgCt1nVg/LWKNMOs4FJJoho4FySgg0uNH9J0BOfO43oBnRDCCkzP5r8la2Oh0sV1/XwmRqVJbnkdckIGK4cje82abTTmK2N0VMsMOyRHq17RZvMaUWPrRgCqZEFSHRNj2tS/QJ6I3Y6eaFXyFB2Z0lTZ2nOSKks+bOQT3db9SGdkG5fLOtEFe122qvqZLZAX6IUt/ex57lkUW3lPvguVoOmaQlHK6JgqAzSNwcHJAE0FcU5WAJik+cp5SjlP6PXwL2qHw+EYcPhC7XA4HAMOX6gdDodjwHEgHHW3G9CoS861U5a3SkxrPq9ZktesN7UsJ+7KMhdqRpJc4iFXcloqkzhPXO684TqalNxlu6Xbc2VX1l2vG67xKVlPMCLTtagfwaCrrEDw6n5GZLxOW47H8lkt/Qt5ya1vG1H4mCe1uF3mm9fWdT3Ykc+wNqn3GbAuOcfUvJ4v61ckB5o/p+tZvpsi/m3o5xPmNU+7DuJST+l+tMZoPyKpx6M6KtsUWvr5bJyV3HJmR38/xYR8f2qGqzM/52gk2+VnuL6t9z06h4gzX9Vcc6jquts0PVcvGtzynuxHmNH9WKPEyl0jCh+ID18x+hFbcjxCVu8flTfkcz5vRXukcbWiT3IyW3P/hqJPVihMAiflvh7+Re1wOBwDDl+oHQ6HY8BxMJ6J7QS6W+QBtSxNitaGYVrnpOnYKWoTI1Wm35aqluUkSEXXnFdF0C1K0yS2jd+strx/ak3fqxnkuc6INq/Se9R3NpkBpUdrNo1oYEHfn8255IougwxVPmTcv0SmW92Q1dF4hCEtU1rfIg/HM1oS1RyXY9+t6WmYoEff3DOeMyVtiIaFDDK1OxNGdMGLWtLYhDzXntJebckdovMMWiP1qhyPxnHDOy4jx6OZ0h1hyqR9SZv6mS15Xbuoze8OTYXMBU0FtZZkG3mcASBOaqlq4oJ81t05XSa7LftRHdbPvlWX9wst411oy/lQuKTLtI/RPN/V98pvyuvqhwzaKS0HLbWrx6OelH1PLGgPxwpJE0NOti+6PM/hcDjuXPhC7XA4HAOOg6E+khGRzOtOjgKt7OrP/OQqmUWHDFOyKs/VZ7UZP/m8LLM2ZtjEZAImt/VQRDqVqhiKiiyZRXuG2UrWdrKifx/Tu+QFqePgoD6pVQXdrLxfQlv2CGRSxWTv3+duTpvNyaq8rmNMn9CQZdp5o80F+cwyK8Y0pCY2c3rsu1miDLQ4AcNn5PiUHjeoByNeuxpHw8uvS5RS1xD8KDrmxtbta0iW9Bzi2EUtg8Lp5OSgWVRMlyi1Tk53PkH3t+YCzykAaI3LRoaSfq6NCVlXaBsDQqqtZN1474jWqc4bbaT+d4t6rQgbpAwx2hOLNNZBP580UbJNg0rNbMrrmpyK8ib5GfyL2uFwOAYcvlA7HA7HgMMXaofD4RhwHAxHHYBACTGZy+0azmhcxvoZKVxhjlqX2T0uL8zrePugoGboFIxEnORpZsm/mJdsG9K70e+RV9mMrqcxTckOzhoyIcNdsZumoPNTmugqXCZe0uDRuW/1Gd2PREPea+iSnj6l48RTWqQs8b2tUd1m9s5j7hsAYlpeN2yMGT+fWDP4X4u2pmKhbjx88jCNRhLYUCGpmdGPQPK3maf1XKyPyzErBT32xfOyTFMH4UOkl4rfAwDI7NE+g8Xb1owLibfu5vV4qPsZvGxune5vvJuB5uKhr+t6Nh/kfugyKd4LMt7NxI6cRIVlPR71aZoLe8YCx+hjv+K1NvRf1OFwOBw/CPhC7XA4HAOOg6E+ukAkUzG/Lr/zy0e1aR0peFHC8I5r6XjlCg0y25U3I3TsIkumFMlzqJE2TDCSAAVDxlU+Iu27kdO6zM6wPLfxuL7XyCl9Xeku8rA0zKk2xTCvH9Jjn11nW1/XM3xOHlfndJlIkjnD2jQ9uxjsvZhoGuY3BbFp6BhAaI3cRPP0Whk91kmmeS7qOVQ50jsoE+esiEYZ7tv6Y7qvTM+wNBAA9k7Ic0whAEC6LOuuzesn1KY5fPSzer4sv0drEZlmyi7r51ybJXmeIfNrjtGcNlapJMl0Nx7WZXIy/wCSFu1E08N6Pmmi4fh9vlqRPEyV9L3yq0z5cd/dM9HhcDjuWPhC7XA4HAOOg6E+ItQOcPlRGfN37KtGjrP3yS1YNm0BoFWXwXISEzrwSzIlTbUWdNAdjEhb0ooRHas0POyhBKDLO+KG6RTJ/N6LOlBRpPYkMtq8amzpfnSnZf9TGd3GFnEfYUSPGXv1hQ39fLbeTm3MGcGdaMzCmL5Xu0jP1TB/Aytutg1Te0y2pzWjxyz/HTlmNSP2dGpcn2tekWNWflD3I52TY90q6Z3+Jql5OAAToHMbxnH9DBMUUCiOa6lKIiPv1TbyZzbn5XXJvL7X8FdkTPGzf9kINFbUrrPJ70lesn6/DkyUpjY2N/W7gGE5RlYbW3V6N43Y6M3J3nQeaA6nV3s/w4wxX7LflH0v3aefT+WIPBd3+F6Gi+w+/Iva4XA4Bhy+UDscDseAwxdqh8PhGHD0xVGHED4I4FcBJAH8eozxH9/8Apic0fXYu9uQKdFxNLRmHEXMvD3dO7dqyISIm0oUNA926I/kddVZzZPu3ceRtQxvLPJSiobHVoK82hT3DaA1ZIwZXWflWkzvkCRr1ohcSF5+CSNSYCtNXnbGmAWKzhZKRiID4mCZjwYAVIiTtWRt1PdEwpBNsfzK4sONMWNpW92IzMf7GkkrciLxz23D4zNJXn7RkCJ2ST4ajSHrEG+bMaLOMY9toUuPLKT0zVKGFyYn7LCQ4DEzPBwTNI7dI4bIk5oUSkbuVJKKsicpoJ+95anJ+wpWrtCF35dawJfv04kdeO+so9a370OeF0JIAvg1AB8C8ACAnwwhPNDrOofD4XDcHvRDfbwdwKsxxjMxxiaATwH46ME2y+FwOBzX0A/1sQDg4nXHlwC8gwuFEJ4E8OT+YfnCk//LK99/836wOAtMAdj4QbfjNuDN0g/gzdMX78fg4XX35RSf+LnXdf+lG/2hn4XaIk4USRNjfArAU7fQqIFHCOHpGOMTP+h2fL94s/QDePP0xfsxeBjkvvRDfVwCcPi640UAywfTHIfD4XAw+lmovwXgRAjhWAghA+BjAP7TwTbL4XA4HNfQk/qIMbZDCH8DwO/jqjzvN2OMLx54ywYDbxYq583SD+DN0xfvx+BhYPsSoiXIdDgcDsfAwD0THQ6HY8DhC7XD4XAMON50C3UI4YMhhFdCCK+GEH7R+HsIIfzL/b8/H0J4vNe1IYSJEMIfhBBO7f9//Lq//Z398q+EEH7kuvP/MIRwMYRQvlP7EUIohBD+Swjh5RDCiyGEm4cOGNB+7J//fAjhO/v9+L/2PW7vyL5c9/f/FEJ44U7tRwjhi/vnntv/z0gte8f0JRNCeCqEcHL/ffmJW+3LTRFjfNP8h6ubnacBHAeQAfAdAA9QmQ8D+Byu6sPfCeAbva4F8CsAfnH/378I4Jf3//3AfrksgGP71yf3//ZOAHMAyndqPwAUALx/v0wGwJcBfOhO68f+30b2/x8AfAbAx+7EZ3LdvX4cwO8AeOFO7QeALwJ44k3yvv99AP9g/98JAFO3c217s31R9+Pu/lEAn4hX8XUAYyGEuR7XfhTAb+//+7cB/MXrzn8qxtiIMZ4F8Op+PYgxfj3GeOVO7keMsRpj/KP9/jQBPIurOvo7qh/77d/bL5PC1RfzVnfRB6YvIYQigP8JwD+4xT4MVD9uAwapLz8L4B8BQIyxG2O8rd6ab7aF2nJ3X+izzM2unb226O7//5qJ1s/9Xg8Grh8hhDEAfx7AH96p/Qgh/D6ANQAlAJ++hX4MWl9+CcA/BVC9xT4MWj8A4Lf2aY+/F4IVx3Dw+7L/bgDAL4UQng0h/G4IYfYW+3JTvNkW6n7c3W9Upi9X+ddxv9eDgepHCCEF4JMA/mWM8UyPuvqut0eZ296PGOOP4CodlQXwgR513VLdPcrctr6EEN4C4O4Y4+/1uP6W6u2zzO1+Jj8VY3wYwHv3//vve9R1K3X3KnM7+5LCVUvzv8UYHwfwNQD/pEddt4Q320Ldj7v7jcrc7NrVfXMJ+/9fu4X7vR4MWj+eAnAqxvgv7vB+IMZYx1XP2luNADkofXkXgLeGEM4B+AqAe0IIX7wD+4EY4+X9/5dwlW+/VUpkUPqyiavWzbUfz98F8DhuJ24n4f2D/g9Xf9nO4CrRf22D4EEq8xHIzYVv9roWwP8OubnwK/v/fhByc+EMrtvw2S/zejYTB6YfuMqDfgZA4k7tB4AigLnr6v13AP7GndgXut9R3Ppm4kD0Y7+uqf0yaVylon7+TuzL/t8+BeAD+//+aQC/e7vWtRjjm2uh3h+kDwM4ias7sn93/9zPX5sE+w/s1/b//l1ct+tsXbt/fhJXudlT+/+fuO5vf3e//Cu4ThGBqzvHl3A1x8glAP/bndYPXP1iiABeAvDc/n8/dwf2YxZXY9Y8D+BFAP8HgNSdOreu+/tR3OJCPSj9ADAE4Jnrnsmvgn6I7pS+7J9fAvCl/f78IYAjt9qXm/3nLuQOh8Mx4HizcdQOh8PxpoMv1A6HwzHg8IXa4XA4Bhy+UDscDseAwxdqh8PhGHD4Qu1wOBwDDl+oHQ6HY8Dx/wPzM3QX4O/RQAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plt.specgram(\n",
    "    train[0],\n",
    "    NFFT=128,\n",
    "    Fs=122.68e6,\n",
    "    noverlap=18,\n",
    ") \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "93ff371e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAEDCAYAAAAcI05xAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO29d7Dm13nf9z1vr7fX3Xv3bkPh7qJyQYACu0ixqNATg7I4jjKKrEE0kTRUXKXxJE7GyjhusemMUxBFsjimKFuS6VCWTJBGwiZWgAKBRd++d/f29vZ+8sdeZJYQ9j2fxb4v8N7V853ZwWLvuef3O7/Tnvp9nPdeBoPBYBhcRN7qFzAYDAZDd9hBbTAYDAMOO6gNBoNhwGEHtcFgMAw47KA2GAyGAYcd1AaDwTDg6NtB7Zz7LefcqnPuFGj7z5xzT+/+edk5t92v9zIYDIa9BtevOGrn3HsklSR9xnt/4gZ+71ck3ee9//m+vJjBYDDsMfRNovbef03S5rX/5pw74pz7onPuKefc151zd77Or35S0uf69V4Gg8Gw1xB7k5/3mKRf9N6/4px7UNL/KukDr/7QObcg6ZCk/+dNfi+DwWAYWLxpB7VzLifpRyT9vnPu1X9OvqbZz0j6A+99+816L4PBYBh0vJkSdUTStvf+3i5tfkbSL71J72MwGAx7Am9aeJ73viDpnHPuE5LkruKeV3/unLtD0qikb71Z72QwGAx7Af0Mz/ucrh66dzjnFp1zf03SX5X015xzP5D0nKSPX/Mrn5T0e97o/AwGg+GH0LfwPIPBYDD0BpaZaDAYDAOOvjgTo9msj4+MBdu5DuuvE2XtSH/RBuurnYDPhApJtM7aRZrsozQz4Ts22mAv14m5cCNJnThqxsEeq3iJjaOVDnfooWgSabJ2vsc7yLVow94908P9RZHYZoNo5uDH6+FYJSleCE9uY5Qt9gicr+h6OdimprIavv66o+3LQR0fGdP8L/03wXaxMpuBxjDbqLFKuL+h86yv4gH2bg4GEg6fZQdwdpmd6Kv3pcPPPMdWUXma7dTKDPwm8PKiB8TMt9jtun53+HZtvzYg9DrILLFBVKfZN6EbOrnJnttO0LkI91cf6e1JuPDvVlG7lfdOsg7h61GBY/bxpWCbCz89i/rKLLP5GvtX3w22+U77S9f9GTqonXMjkn5T0glJXtLPe++vG53h2lJiJ/zRsNTKmikevrSUXmM7Zvt2doclttnbEWlPknYOpVC7NmjWzDLxMVZFzfClREGl0dX72UKJ1cJtWln2TIrRl9lHWb+L3Ur0sKFzES+G+6Oa0uQzbO+sPcwOYDpWCqpBrr5/JtiGavuxKhT8PvFAsE3n8T+9/nPY6+jTkr7ovX/EOZeQlOnW2EfYIZzaYIOMXkHN1MyH2+wcgioNVH2pSYNeSqkttkKSW+FDuDjPDup4CTVTeo3NV2WWbUD67RIF1i5aD79fLWyRkyQ1oJRZ3s8OYLrx2yn2jdMr7P0SxXB/UbjWN45B4WWH9UcRaUEtI86+CREQ5r/IeOGKR8ChI6k4H14n7S5HU/CVnXNDkt4j6eckyXvfkAQtvQaDwWC4WZAr8rCkNUm/vZug8pSkT3nvf8jQ4Jx7VNKjkhQbGkV2ylaW3YDUrkhUOOo3SW6xds0ca+c8tMfnmBRMTCmxCupKrg0l5X1sDPS5VMvIrDFxtDoefr9oA46hzL6Jj8I1DMcage9HtAdJ6oAdHgUmI0nKXoFOXWa9U32Umj7gN073ThvZOj7Engl9BR0y/122PjkVYpLul/S/ee/vk1SW9GuvbeS9f8x7f9J7fzKW6bEh0GAwGP4Cg0jUi5IWvfff2f3/P9DrHNTXwnWYXbHBLi0loLMrUQAebniL43A66NSJQqdDHLYjYWZUY2nmoQOLhiJBCS0K57W0D8bUgU9HI018hH2T3CKT9iuTbAxUUi7PsfdLrYXb4LBLGM3T6urBugZQoI6BIIEbea7rhAdSnYCO+Br7KBFynnRZSsGD2nu/7Jy75Jy7w3v/kqQflfR8199xUiscPYadU90GcC1IqFQcOqaoNze5DdXBDIwOSfUwBA5+XmpaotEhUbh4a8BUIXFHXLwYbhOB3hV6oMfL7OX8LNv4dbhOOlF48QNHMT2o6Z6oTLOx0ufS6KAsDToAAgwNJqiNsfkiYXzdhD4a9fErkj67G/FxVtJ/CX/PYDAYDDcJdFB775+WdJJ26tpScit8g9BY0GYOOuJGws8kSTGSlL9EkyyYODp8jg124xgT5UjMMM5yg8isQOkRmg0oaLIIkbyoVkDV8uIck3XoWqcOsdQGe8HaBOmrt5JyHpqDyrA/KlHTMFOiGVATVOEgG8PI6fCGjdav/92M68NgMBgGHH1JIXe+t1ls9HaLvH6a/A+BhCtJUmmOxVNR7ohoDRpaHUygAFJmHIaYuQ6TztLrbFJX7+8tT0J6tXdp/wlgx5ZuIJwO2jKJo1uSOjDcK1bpnW+EaqzURt1TP4skB79xAzrFsyvhdTx0agP1VR9mWZjFA2HNu33q+oeJSdQGg8Ew4OiLRN2JMwIfKo2m13vn4aZhd9UpdjtTEp3CAvvU9JuQsCAaLZFbYpLy1u29pc+jiTE0OoAwIxLfiSRt38GemT/H2lHtAYf7TbGFQvwKm8dgqN8mDG2FUT8Nln2No8NomClJKqN8JdlVNl+E16SbhtEf00eHHZo0q4+CHJrNod7GDNMN2IbqIOW1ICotit2UVB+BDkyo+vaaNrUMuUOIUywCszCpOagOQwzTKzBkcQyGtsGdi5xd0AZBzTyNYXrws+dml5kgsTZNidTCbWg4bSnN9g4J9+y2H8z0YTAYDAOO/lQh94zxqjnMupt7gsXd1KbDWTZrM2zIlGeWOp2oBEQdp8QBNP4sE6k3TrAQwwbURjCjIEy0oSYcIgXRzFRslqFFLWjFO9hu+DyTMncOh18wtQ4zWKEGTLm86fxXpuBHpgmsoB3drySxT5I6YKzdnmkStcFgMAw4+iNRO8gqBgvr7tzGSJ7aScCeRjmQYfmnXqfLplhUkOqj4TbFA0zcp9IDlQqpbRwzwMGQNSKhkYILkpTcZO2wtA+fi0HnAtiVaagfSdGXpOIhylDI+qMJOTQkmDjiaZJNB3JgowS1HqSQ3xA6Uak2Hm6XXoVE31Dur00A6k+4OChZeRMSBdL4XRoLSsjZsVkGtqMRLvTwos4pPhfhb5e93Lv6izcCErsrSet3QdPcWu/ej16EkKkXH5jUbEAjsFoZNrcjz4c34/K7mF125AwbbGk2bL7ptm/M9GEwGAwDjrc0PC8NCeFpphNx7AxdZPF0m3fAuGfo56DqJTWRkEy32mhvS3HROFWqAVFTCo19Jpj50mXU7srH5lC79AZbw7Eqa4dZ26BTNLET/nbl/TBnABbTIOFvEl/rtM4lNVdsnQjzK1NHZxGWYpt4Nmz7ONfFJGMStcFgMAw4+uNMFLNTdmAZI+zsAs+kkjK125LK5xLnXs5dZpJXHEhoNHmiDgu+Ui7vOCioKknl/aw/ysY3fC78TQr3zaK+sBQHpceto9CxC9cdLSBMnKK4ZBv0FeRgtl59iK3PwghqpiTMnMyuhFNYG3k2EdEm5KGZB1wfz1z//fsW9UFUh/EfsHLFiz/GZoqojdSBQc0BNHKhBQnGqTOxPBueOkoENHQWNcMxw7TqTWaZjXXkDLMHFA6EvwmtcUhMBpLUhE7HJkyXpqYPwXbEvED3BDV91EbYAUwzGKlzGse0gwtn9GUWulRYYAc6+cbdnLVm+jAYDIYBR18kau/Y7XblfUxSpqo0yUwj8YySlFuC1ziMBXcd6NiD1KRVEIqYv8jGUNrHRBHqOKNOTJrpduVhtkzHT4W/HXVMU5WWhARK3HyXvwQLB2wytWX7SPjBhMxK4ua72W8yW8rKSVbkEFe1h87JynS4IY2PJuX/JKYpdQtMMInaYDAYBhz9KxzQw4rQ9EatzITb5C9AmsujsMTOK0yyGT7LxJbqJBMLmvnwOCrT7ANTBsD6MPsm1DZOGQVpGBdhxmtmoWwC1xwtvEylVlrVeuM427qEoa4OpUeKy+9hknL+Evt41XFYwR2WWUsWwnt2Z4FW3oXPvMnShH0jZSK16WhFaOpMIJcDzTij75ZZhrUV74EeGwhS8p6mLY+/wDxTtVF28I99ewW123rHNGrXibGNWpkIt4vDeHZaXZpSElBhozbSW1MKMUHiKuTQbNgY7V3cu8Sd0zQqZfOO8ICpEIkvYEARbJmJBoPBsIfRH9NHmzkAqSOGSjfDr4RFbyxRQ3PAzhEmttKqF9RhF2mG71jK9bFziIlUVEIv/jSLVZ58mkny42X2TTaOhZdz/lk2sdUJtjVIlW9JGjpL6WtZfxSEDpdKhXQNx4swxBRUgZJugEhrm3KChNtQmmNq+qBV7a8Hk6gNBoNhwNG3zEQCmgRAb600oAilEnUbmpQpU1gK1n3cOQRL+4CZo3b2CEzayUHmuR3oiN0+0tsajMSXUZmkWRGwGZx/KilTW3YU9kf8NlkYikprK+ZgiCFlxZv9U+YIWL+bEXQQ+3MSOsR3DrG1nr94c3b7/sRRx6Q6MFdQfluaVtsG3mu6YSjBCz2A6canlxdLDe5dtRhJKhyEWX0wrXr8ebYBV06yDRgFzq4GdNbRdUJpU6l5gRIkTUATThlE/oycYhO2eQyQoIs7/+mltPQwm//0Kqz92Ap/49QGM8uVp9m7tQmVrGUmGgwGw94Fkhudc+clFSW1JbW89yd78XBqXqA3by/RTrDbmYQhSlJyh4kZhQhTzYmW0W5DWlJoDaD8J7VJ9u2oqkqcPxJbJ5TDI9Jg327sBRaztvIAW+yUIpbGUZM5W/wwY+WiPCR1WAuVxlGXZ2AVJcjjQr7xzkGmxg+fY5rN2j3h+erm/L8R08f7vffrN9DeYDAYDD1Af8LzWqxsU3Wyd7XwJGnoYtizU5tkQ6ak9jTEMLfIbt5OnEkPrhN+bgNKNjSRYeIUa3jhw2zCqtPQvlvtnQO4CEM9qfawfjeTlOtjcKx1GGKGy1iFpdZ4iVKmwrC7/b3lv6G+FkqbGyGaJqw7Vlhg50k7BcbQAxu1l/Ql59xTzrlH4e8YDAaDoQegEvXD3vsrzrkpSV92zr3ovf/atQ12D/BHJSk6OqLNu8M3eeYKM45W9lMmu/BwaFknKilT0v1Okt2JbVigszEEIlyGYKmzChsrlZTbefZcH+kdK6IkpV4Kr6dmDlaXn2Xtkpu99cc3htm3o5J3JxvWMtsNdgyU59i7tXOQtZEyGVZoSbne+WRo4Qhqt48Sn0eXJYdmyHt/Zfe/q865z0t6h6SvvabNY5Iek6TkwTnvM+EFQuMoKYgqXWX0Etr3DWaqWH07u2zW72IxwzT7iyyQWImaUVAzNUdg0DBFlI3VJ9kL7twOaG6hGYWiNsHeLb0KL2po5ovBQ6mVCa9PmoVLLzlKVEX1eXrwp9bYXiQmstoke2YCXtSkOlK3mPzgU5xzWedc/tW/S/oxSafQ2xkMBoPhpkEk6mlJn3fOvdr+d733X+z6G22n6A4oi0SFG2r5ABKajzOpYOUBdjtjaRTQkko80606DZxEkHMhBk0f8W0oFUJuhg5sJ8hQR+aiOcbEx9g2rK0JpX0yX5LUybP3a9WZhpa7ADhhoBTfhlm9HdhfchWGoibZ3qEsgAQOhrbSd6uCRDbf5f2Dq9F7f1bSPehtDAaDwdBz9CU8L1HwOvB4WDLYONa7wpCS1E4DiRoGxVN+EXyL00xzqj3Eww19lEkslOif5sHnFll/dchvTeffxwBHN8zuoaFoNUhqL+g4dWW4JeEyrk6RPcH6ylxmY6VOx8Yoa5e9xJ47fJ6po8X94QHPfIvl/K9CeoM2GerNOhP7BRofSW0kpGJIDMbHUtBMsjjMrmxAro/ERnixJXZ6G6feGKEqKHtueo31NwLoayVp5YHwZAy/DPlKijByxcGsOfhNaCECmmFJImZGzsDsunvZcYEjUuDpQ52dpdne1f6kWbO95DWxwgEGg8Gwh9EXiTrSaCt1IczItXX7FOqPcj3kzwPVF8ZHJ7d7y0NAWfYoX0EKsOdRe0sj37tKypIUL7N2DkpKtKo5IY6vMwI4TT3FVK/iAlOBqORFvx2VvEko2s4heAxABTh3gbWj5hs6Z4kd1q60P7yepr/HsnDX7mMecWI9uKnwPIPBYDC8teiLRF0fjen8I2FpefIHTKSiTGGZtbAzYWOa9UXZ00iVZ0kqzfU20YI4gOJlmIUHSyLRcEpasCAFGQV3DjLbI+HnqEB+kY0TlLicNaPc61FYxIH6FYjWgotH0yIJIGtWkhLbMOGJrjtY3DYJ6Le3j7IPPPV9Rp8ZrYQn4kyXknP9cSa67jGBr6JwsHdUjZJUmQpv6PxFmFadhCnko711EtEY5DogwKcbix42NKuPOokzl5l6uXGc5fOSyytR6F1dPYlH/eBqRo6WlmHNhkAkxMaJ3l2EklQfYe0mnmGCWmUf+8h0L5ICA7Vx1lcjD99tPnzwt09f38Bhpg+DwWAYcPSH5rQjRYFGQEPbGkOsHQlFogQ/C19gnomVhxmXKJW8sNOpFJYKeilhSFK8ytoVDtL7n4mtpMSWBMO4cP1NNhGV2d5mf1LHGXU6Imc3XXNF6JweZmPdgRo1nX86t+TcofuwtA9mMIN36/ZeJlEbDAbDgKNvCS/k1oo0e+s4IVSCMSiJ1PYxaY9KGbSAbGaZXeV1ILVQO+v4c5Ap8D6mFlBekzYKMeTJHcRu66ENOLnFvkmkxbJrSSENifNpxKCjmGiQtEhGcYG1oxIwHWtqo3fUvxLbizRbM7XF9uv2beEOu7E09+2gJpNP6w1mL7N25GCikRA7C+xQGjnDQhyKB1h/O0eZkkNiRmn0RRs6TqkDK1qDUSRw9dFszfHnw4fr5p3soet3wQN4AzXTyFkWhF6A64RWRyIHDnU60/mimYT7nmA39dqDva1+Ti5+atLaPgpP9JuEmT4MBoNhwPGWmj56XThg7vGtYJvld7HbmcZkFueYBDTxme+jdpc/9XbUjsTHUodTO0EdYqy/9Dp0xE1B0nXqOJsOSzeUryQBshwl7hBf+pHeSuhUeiREZblLrK9Is7cZrGvvYHsxu8RE/tW3w3BfEtMOzUEtWJGJOpOvB5OoDQaDYcDRF4naR9hNQ1m2hs4x8WHpveEbmjowSfibxB0Y9fffjdpRe2EKcJGMPMdCDBc/xCSbsReZ8XH7Nrascpd75ziVpBZwTlGtgEqsoy8z8fHKe2FGLLTv0sSY/V8NOyoKC0zanzjFxrp9lGmZVJOrDzN5kkq32ZXwJmsMsfmiDvubDR01idpgMBgGHG8xHzVsSAPZgQO2QlNDoaQ88SyTMqgts82EG8R4t/YAy+VtDkGifyjZ5C8ytYB6zGmoFIpyoZF+ULO58m62hUafZ/21IIUA5b9YeSAc20olUZouHYMFmmma9uQPWLzf5glWYWLt7vCcDZ2H2h4cA+Le7tJVfzIT24y0nqqXhcPsgCDxu7jGITxYCwfYJ8yusAcnIVENccRmL7O+Uuu95TWpwizM+ih0xMBwP3LhtGBF6/Tl3tbzK81BOlwYM5xZhnHUgNbXR3obTtkCBTwkKQEqc0vSxnF2AMchj0sdVI6vb7H5olS95DK0zESDwWDYw+gbex4R9WnGGeU/IGFBNOwKB8/Dkk1VWFuPmlzaQJKjYVJUjW5CzhXaXycBiynAcRDTR6zUW8m2AVgMJakDTVqZNVhvMMvGQWom0pJt1CxDQyDp+qRmKMpQScxfDUbhgyhTJVbVvJvD0SRqg8FgGHD0RaKO1qSxF8PXYGGhtzy4hE8htc2u551D7N2qE+yuww4xKGUQKcjDytdUiislYCFXuKriRdYf/SYTp8IGw6V3sokYPsvy7yszjIgmGc7FutrfJPsmtTEoBWehathDUO6Q1Ab1jUBfBqSkIHb7+S8zA/rmCcZvMPFHLwXbXNi+vtO0Lwd1J8YWXBR6h12nd1XIMWBX9ABuMe57XPfNATVv7Hl2wxWOwJeDyC2yeY3CmHbqnNp4W3g508tr5wizVdBDiVL15hZZOwpyGdI1R6l66cU6/hxrWJzrbUwzyRK89CE2YSOnmeBXeP9twTbt/3R925KZPgwGg2HA0R9nYoRJt00oyFEJjcRlNjPsbqIx3pTZjcbvUkpXYl5Yepi9XAaGDtJYcCrZCJb2omGBESDcpFdgMYV1Jim5FnQSQ6djbaK3pd1ICBzV9ih9aXKLxuUzdbR0gD2XmkiHT4fXe3kfm9fSfjYGkpnYTWMxidpgMBgGHFiids5FJT0p6bL3/ie6tm1JKVDKKF5i0gPlho0QKYNKtjCcaugCk0bjXSoMX4vaKLuhC4cArwV1rkACdxpiRm2U+UtMLNw5wuLCSKhUepXNA3Z0Q35zXIoNhqLR8NHaBOgLJm204Rjyl3rLnhgBBUEkrgWX5sPPpfNAtQcadns93Ijp41OSXpAUtrI7qRMHBwl0JlLPHnHsleZ6p0ZLUgmqSB0YMZFegyXvQRQBLRyQ3mSDrU5SsiXqJGJ2Hh5FABr22ElMY8bpwdrr6CBSD5M64ampkuYM0Ati7IXeRmrFC72rQj50kQkbyw+GhY2bzkx0zs1J+nFJv4neymAwGAw9A5Wo/7mkvy3put4p59yjkh6VpEQWphL2mCCnBSgBqMQShdIoHQM1G9Sh04moXJVpmoXJPgqlJS3tZzoyIZaSuEpbRY449sz5f7+M2l34xAxqRyVganKjIDweuCYl/HZ1GOOdv8jWEzXN1SfYOOb+j1PBNht/+QTqi0jKEsvW7HY2BSVq59xPSFr13j/V9SHeP+a9P+m9PxlL9TYu12AwGP4ig0jUD0v6KefcxySlJA055/619/4/v94v+CgzntOK0Bigu5FXIJcClPZIlWcJOjqlGwjj6x07YRJKVDQLEzvYYChiap31VzgSbpOFZaeufJRJyjQkDFe1hpXZifYoSfXx8LejGgsN46T8N+snmEKfghXcKdYfCUvL1PnbzEKGylWwX7vY7INfynv/65J+XZKcc++T9De7HdLSboUXsJASwKgvSW14oBOiou3b2WGTu8TejTpYRuEFsQMpXcnGpxEpVZi2TFPD6UaNQm8+Ju8BMc30Eqnk2DNpRZZoFWbhwjMpBw+vnVT4m9C49wSsekTj7Wm70jxrl9hi66lwGBBVQcpUujaJo9tImQwGg2EP44YyE733X5H0lVC7q4UDCFE2u7WoyuU64XuHOmtoGB+tkF1Y6O2duO+JcLnqzktnUF9b/90DqF3uImqm8gzT86kkN/k08+zGquHJpVmONFsvBauG08xEWlmEhgUSzasGnXDRGlvD5Rm41uEY8udZuybUgnwUPBhqNhPPskiH7KXwQRGtXr8vk6gNBoNhwNEXro94oanZLy0F213+8X2ov04MlvYBUvzIaZaud/4nmLdm+Ay0Fd4GkwCgzat0NMxsvvHxd7BnwvDH0ZfYt1t8P/t2tJzUzmHm2SHzT52/1H5KnU5tSLrv2pDSEzIPkoHEIOdKZp1J+0VYdiyzBDl8JmGYKfUXAM4SGopKHeytdFjcb5+5vvrTl4O6MRrXhZ8OH8JRmOJchRM1cjY8U+t3s0OEOgkq02yihs7CRQljUKtjYZ2WHja0tmK0xEwQ7RRM+YbRJo0cNEMBhx299KlJg5og0is045SdNvVRtnVJBit1EtMU8rnPnkbtmrcxQW1lmNnIpr7PWKMqM2ETGY/6gGRbINOxW66FmT4MBoNhwNEfmlPP1Gl6a8VhWFB5OixlUnIUSuk4/7mzqN3lTxxG7eJFGJe5E1bN2iA0S5Imn2TM8asPskJylEq0uMDajb7MpMydQ+HlTGlkp7/LAt+beeY4Xb+LbbXGGmtXnGfPJes43WNioYs/dxS1o3UpaZbw5tvY5JK4cczNQql/Sda0VSE3GAyGvYu+SNSRFmPt2mEXLy/FBS5USv1Jy4Rd/FkmKedh8gm9oTeOhSUqmrSz/K4R1I4WSaCk9nSszTRMAgLNaHhmYYE1rE5B5rk8m4vSLAxtpKXdgIZGs3DrkMKnnYGJMSRMTpwTpDbK1kkdLHfy3SS+1pMg47SbM9QkaoPBYBhw9EWibiek0gHARw05BiivAbEr0WgOD6MDaNIG5ehNFqhxLNyEeqSJhCExqUCS6uOsXQwmC6U2mY26tD8sBVPbIw2no8TxTVjclqaQJ0CRBIlFEdE9EYN+G1oQhH7jNuC2l6RIC1JSJML9US0jRcuOgYSnbi36Y/pos4UUq8BFDj8a2oSUlpSGLCWg8w9O6Ni3w/HnktTIh0Obpr7LSt4vfog5CSnXQxRu/CaMaV5+kJkhxp8Le7DX7mMXJl1zw2dYEDqpKiJxk9vwOeZhW3kgbA+kgtD0k+yZm8fYfGWvwJBVWEdS8BKeOBW2V1DHZGUGXiLg03X7Gmb6MBgMhgFHXyRq75haM3SJSSP1BqxWDPg5KKMcTWSoj9G7jkkPix/fj9oRyevix5ikTEGLGlBulrHnmWd36WFmXyLhmVTNbwxBaQ86sBJF+O3WmJln6zYmtZIQWJrRt3UHe+bs15kmt/Z25p2mIZV0z24fDY+DsmJS8w3ym1p4nsFgMOxd9EeijjImq7W72eOxgwXkbVAHW2aVSfvJLXbXUU4ASjBPwgyppNSC4VRUZEkU2LcrHILiCER6k0jyMIQLsuzllqCIDrfa9lHWrgU/HQlZnP0OizHbvJOJtpsnmKRMwzNTsOAzTaAj1AU07JI62NG7dXmt/mQmSux0hQUB6ITOfjVMbFA+yBZR9gxT35oZFjJBD+AidDqRrD5KelSBXnVMQATjnosHqIONPbe0P9xfdgmS7UAOl9oopXSFAsIyjBmGUUSkOMfyg+wAHn2ZXcCVKeiwhUJTFkZCpZbZ+20fCS9kKuTUx1i7iWfC79YtJttMHwaDwTDg6ItE7TosljIG683R+Mi1B8LSLZVsKlMsDQvTYcIsLIoWqNU2/fV11Ne5T0yidpRzoQJLe1En0f6vsoD78z8ZjjMjxSUkyUcgv8wspMQbbCMAACAASURBVPSE2k0RFpigoW2EcycH2RNpQQg6r7RwAKkPKknlacinApRqmsHcTrNvt3ln+Nu1v3n9n5lEbTAYDAOOvkjUia2m5j+/HGy3+r5p1N/E00z0vvTB8FVJK1+3IJk6lWyqU+y59IaOlcPvt/7gBOqLcBZLUqQBQ9YA9+4NPbfODIZEetz/xyuor9X3sAkrM0pl7JyixXJpqGQW2LypBkQx9wWWtHXlo7OoHc0SJGOVpDjYOzQJiOxDifuorvucm/v110c7E9fOvWF1msa0Lj/UOy8yraCCHWeQ5IdmRFKMvRhelFVYhIAS0NBMQroo6TfevoMVw8tcDr/f8o+yA5gSBtGIJEreQyu8NIahCY8cwtAEQRyTknT2v5hB7ZK43iRrl1+E0SFgz9I1nL/ELgdi+ux2Hprpw2AwGAYcfZGoOzFGQkSlkeoMa3jo88Vgm6V3MXacBkzqwwTj8IbOXGHtEjthHbmRYyJrFJo0Ulss/KkxzJYVVfOToEiCJNVAlqiH4mMMlPWSpHgBlok7zVQ56hCb/DNmDjz3l8Ipdqk1NobSPKUbhubAafiNYVZnfZjJnR2wF6kzMbUBQwKPhvditzPCJGqDwWAYcPRFoo42vPKXAZMZzEykDHUrD4WlZeokiMPQwTq0A1NbdhbaizfvpMbxMKhku3GMzRcN4yOSjSRtHGfPJQkZNBmjvI8mAbH5al2EGZFwPW3dycgoInVArwm1QkpL7GEiG3acw4SnVhr6UMhygmt44zjTWpFdvMsS6ctB3Uo6bR8Jd90YZSskvcwWOYmPpA4saqqYegqqtLPsU2+egB7uxXCbSBOqoLDKO3WI1UdhhQ84F4lt9n6FgyBWFUb9NEbY2ow02bull9lps/xORkBFOJUlafy58Dh2DvVWsSbRNxI3uRCK0BsBYUJowmpG9N1IxSur8GIwGAx7GEExzzmXkvQ1Scnd9n/gvf973X7Hx6TaBJCqIMkPrQ9HJD5KQETzCFfewcTC9AollmHtsoBKdPtIbzPJqIrcHGYNYxVIaDXN+osCNT++Q9VjOF+olbTyEFvEnQTr0cG908j1rsILrVLTgPPfhpVgMrCqfZyGmYK4bJxd2cPwzG7BFUQfr0v6gPe+5JyLS/qGc+4/eu+/zV7RYDAYDDeD4EHtvfeSXnWtxXf/dL1HXFtKbgEnBnRi0YQMkhGX2oBOImi9p1wf7RSUCmC4VwlkxFGbMrGfSVIZFGaQpOQ6k+SxzRsylMWAhDZ8Dkp7kAGQOIkk7kxObEPukKXe1eqj85CAnCDUokoTXuowVJbW4CTrnZb/olI8PU+uB/RFnXNR59zTklYlfdl7/53XafOoc+5J59yT7Qr8YgaDwWAIAp3z3vu2pHudcyOSPu+cO+G9P/WaNo9JekySUvvmPSE2p0VQd1hGqlKALG72n3ahqLoGy5/6EdSO2m1TiNReSm1Sdj8gKUGPNA0Jo7zQcVgEl4YFRmvs/VBI6D1M2qPJWESKl/hc0HaxOnvBErBRJ8J5YpJ4NMfsN9kgLr+b+XciLejLohQHZAn0mPKB2MW7RZrdkEDuvd92zn1F0kcknbpeO+fZQqfqYKTRO2L7y3+HHcBUpaXqFuaO6LAVkgRV3ukBjEE9Z7TSOwyBbMPY9/pQeAfSi5DW30NOc0lD56CjC5LpRxt0MkC2JjTfjZxhN3VpP/PExSDxGcX0d1ixj/M/FY69S0CnM81gLAPW5G7zEJwi59zkriQt51xa0gclvchez2AwGAw3CyJRz0r6HedcVFcP9n/rvf8PXX/Ds5uGGtgpdwBJAshBhq2do6gZ5iGgrGiFg0y8aeXD/aXDTLOSpNICk85GT8GQMEjBSZOPqLOzMh1+LqW5zUMJuLK/t/OagWGcxf1s8xBpmVbcXrubqTYtlrOj8eeY7WvtXjbWy+9n8YOtTHi9H/wCswctfpA9swn2602ZPrz3z0i6D72NwWAwGHqO/lQhj0nVqfANQiVlmn0y9my4DSVcpwkP0VpvSzYRSVliNtTiYej9gTbKMigeK/EEilaOjXXuCUa8snl3WDSkjknK95yANm/qsCscYe1o6nKH7jHSFyyCTPwnEpeUcZJaG3KMAIn64ochyyakwYgSe/ybXYXcx71as2AlFeDjh5iKtH1H2APYGmGnSPYc08spJ0BjGFI6QicGubyqM2ys8YvMHlA/wpxJvgLJm0rMm3jxx9hHjlYAnwI8t4qHqLOOoQFjwTsp6ExeY98utRpeKKWD8EKHXsfSEDRV5uFzYbZmK3+TZVSuAY6PH2ZjiFZv7qg1rg+DwWAYcPSnCnnEK5EJS9StNXZtJXIsdaqRCN9urgnpJkegWjbJpNboFqyQDKXgWCasZcSg+NiYY3p0Ms3erR2H2X/Qs5c+zeZscz48/9EyzDhMUu7P3prvKGoHmCRXj4bH4aG0F4VZvUjNl6R9TENrF5l2O7SP2ZeKF8KpjvV5tifmv8DW0+VHwmP1XdacSdQGg8Ew4OiLRC1JnQ7IiDrEbsDqDkhzlBRNASmjxG7n1iizi8dXWX80gL6ShIx3wG6XH2LR+M0S02zqcB6GJpnzr7LIwr02T7K5iAIto5NiskkMaGeS1Kqz+XJFWJ5sgmmPHjrOCFyOaUp1yH+hYehNbsBvB7XgTJJJwYVMeG4jcTb/K59kz0xEgP+ky+ftjzOxFVFrPbwJs/M7qL/WCnTsAQL8yDD7sG6NqeW545uo3dYS8yK7FoysWA0frslxdhFmx1npjsol5tSLUI8dxMgUG0ehEF5zUXgAR4DJ4Gp/kLxnln3jVIIdco0WO+Qqa+FImEiWPXPqCGNRanfYGt44D9L1JMXGmYmk3qSJGSCdm1IwN9k8dIB5qdvla6YPg8FgGHD0x/QR8VIurIY22+w2amUhyU8q/MzoIlPfmxNQfYNIjECHaIVpD+1Y+I4t15hJowMlIOo42wHOGknyI8ykkU+xb7e9kQOtmESdJGY0SR4qDyM5JlFvl1haX/MiSyd0II46CtX8bIJpo+UGW3fxSWaaa2yxPbu9wdoRhc+tw70zzNawq4I91sVcbBK1wWAwDDj6I1F3nFQKd12JMDtwdpHdJ5XxcJv8cWZnqzWYZLuFpDhpdJw52KjNi9z41TRkMVtk7fwok7yGX2LLaucuaAeOsHaZ4bCEBgtkq1aFGQ80BBLalBuLsLr4LOScBY44f4E9cy3FtEzq5mwUYXguLETRnGeaV3Yo/O0azzGt0M2xZ7a2wTd+szMTXayjxFRY1auXYbQBrGodBQ6gIahGb20wx1l+lKm0W8uw4Bz1w42DTQMzBD1cBS7L1Lyde+EBnKTmBbb1q+XwhRNPsjH4VXZ5RWeY+r5TZCaNyDQ7gB28vGIgHrwFyKwkqVph32RkiO2JCBDmJMkfZVzCEWjCc+BypfQGSRDNIUltYuaLXr8vM30YDAbDgKNvcdRECooD558ktYbYa45kgUoDHZgUVNqLQmm0vQO5P8Fj4yNMOmul2DOjMVghGzqnaH+FGpPkUiAbNptiDrFND516ZfbtXAWuOyg6ebieRsbC0mjlFWb6aEyzZ5YTTFOmFWPyYF9L0uY604IbsfB5EitD8jZaYaIBJraLcG4StcFgMAw4+iZRR4DtJp6EGVGTzA5UrYdv8mKJZcMls0zyKm0xyYvaY2Mj7LktYH/OZWBYW4MtA2oXbcH+aFhgBD53diRciunSGkuy0BQszV1mY02twIzTu5nTuVljz40Bv01jkq3N1CiTbOMx1l81y+Z18/IIarf/ICiaKml5M+wv8gtsrDGQgS1JItwxN1OKy2AwGAxvLfqTQu6lJpCqcmkmtdCEh6XlsLQUWYc24AVatZY1Sz3DJPnGvfC5wOZVrsK6U1AqiEPbc+cSs3lGD7CxkrUksdTl1iZLilAehqKlaXEGyAmTgL6Mdu8iHCij4PABFuFCU8iTK5D/5DgrWltvwf7AN3GQAbJZh8yDxB/X5b36Y/rwTm1wkGwVmNlgfnILtSPUqg1QeUaSEnG2YWp1dgDX72EhS5hqB+wFGpPt63BjjbFv0tgPyaCo+g75OS6+PB1s4+HBOgQJrQqb7FJqvo3Nfx2EGErCAkIN8F+4Fqx6Q/lFYEZs/QATwBLwUtopsr3YBnw66edYXxF47zdz4ODvEvNupg+DwWAYcPQn4SXaUTofvi1paEupzqSMPHCerW+ym7KWhGFXgNNEkvI5JqEloCOmDsZaKrPrnhadisLg/qkxpqounZlE7ZLn2Djy7w07k2rfmkB9ZfYxLa45xKTMGkzuomXMJuZZYULCUBedZ2tz+yJz6mmI0pwyObFRZ/OfnmBaC9HQz5ZnUF/REtRGN8PtIl2OEpOoDQaDYcDRNxs1sY+m0ywUbRvasknqKk4CgaTmHvJH78AxxKEzqQPsdgkY/ki5l7d2mD2WJu3Ex5kkVxuFZP/AidUYYVrB8mUWxpcdg5oSLGNWh+upUocaH7A/03UShbwW5RKTgJNXYBEPyJ6Z3g9LykUByyaUlGlx2+pk+N26FTbuXxw12Px16M2n6PxRmJUp9VGmMtahVy9yGppSYA22Oq1mAYhlaBbe2gojoMmOsEMpD00fK6uQDhVWUSmshg8In2GXUgRWZKkkYN3HLDvkYml2UVeL0IvVhT/iVVTWmRBx+MgKaleCNLc1mOkYJRShktIwAODFS8CsAehhJSk5zAQ/UvHKdTEtmunDYDAYBhx9kaijkQ5z7K0wRrnYBlORNh8Mq3AxyABGQngkyS8wqZXSYXa7Va8Fyfxsw/jo5CKM8Z1gJbFKkJsDA0iFkhSZCks3LcjNEYVOvRbMTOycZRJwewKWChtl6y5aAOagaRgfD53/MViFvO2heSHH3u/KKnN2xkA+QGSamYOICVJi50m3IhQmURsMBsOAIygOOOfmJX1G0oyuRnI95r3/dLff6XiHnB0Osqe14I0aBw6b9HeZQ6x4BGacwfJUo/tYId+tK8y+lwDJJxUo2TYPMTsbDfejiTbJDJNaakWoBQEHMM04o1XoU9A5XUtBRjnoG+lsw1JRxAEICq9K0vI25FSH8Z7pA0xDK6+yPTs8zMLzquBsykD/TgtmYRaXAbNfl+K2ZIZakv6G9/77zrm8pKecc1/23j9/vV/otCLIQTEGDy+NsWakBmPpfvZho1DdHh1madDrkIJxZoFVoCk9Ec7C68CCCz4P02Xz7PA6fugKaldtMTPEdopdEORiorULx9Nw08Mx1PLQRALNC2VAQCaxQhk09Xr7eVBCSVLmGHPYN2DV8BSMDhrLsjk7uxR2JrZKMMIJEktFK32umei9X/Lef3/370VJL0jaj97OYDAYDDeNG3ImOucOSrpP0nde52ePSnpUkvIzGX385J8F+3vf0IvouRGoS31p+65gm8PpNdTXycxZ1O63V9/N+jv0bdQuH2HSQ+XngfQYZRJGwzNTxWab1YccizKqzqkYU32/UbodtZuOh8MC70+fQ319+sqHULtHZp5C7e5PXUTtvlq+A7U7nGShcjPR8Dd5rr4P9ZW4jZkDMxEWiljpMJPWCzX2fsfTi6jd+KHw+vyT7XtQX3dl2TNnYmHrwad+6/qZtdiZ6JzLSfpDSb/qvf9zs++9f8x7f9J7fzI92mOvv8FgMPwFBpKonXNxXT2kP+u9/3eh9pOxkv7ria8G+610mEB/NM5srSPRPyfo/zn8yjOfRH399P3PoHb35C+hdvemLqB2k1GYrQek4AXo6KTodIsfugY1zzSgfITN/7HEt1C7L5ePBtvsg1rGX9/3JdRuu8OSRfKO2ffvSrH1dAfQHiRpLBoWmvIRuDYjzH6egZSuFO9Ks2+y3mbPvT0eHkdq9EnU12fWH0btfnXqiWCbRJfaZM4HNp9zzkn6HUmb3vtfJS918p6U/+7j88F2dc+8/jEx1bzqw57azQ7bMP938ThqF4cb8KE0M6UkYSG5S61wdMgMNC3si7Jn5iPMgRWBiloEkrpGHeuPrKeVNlPLqapJCa024SHy2a2HULs700uo3fsyp4Nt6FhTMCIlE2H7NedgpXc4/00Po8McCDrosGieIjxPnm+G9+sv/dR5vfxs7XW/MvkCD0v6WUkfcM49vfvnY+jtDAaDwXDTCErUbwTH7k743/0P4fCxw9CVmXZMkmspfKNS6Zz0JUk1z27UlTasD9dmoWhzsbCJZLHFeEi+XmHOujiU9j+Wew61y8BsTVo3nphScpDpnWp7VIqrwHZUul2BGXFfLr8t2Ob+9HnUV9ZBErUOW3dRWP0g5dhcvC3B3o9I8kQ7l/jZRPDQRxb11A9en2XIMhMNBoNhwNEXro9SJ4XCjJ6OsmSRd8Mbf6cDMo4ikCMB3vbD0MESh/1FHJO8Sas2tAHfkWT2zv0xlsjwG1c+ito9MsEcNgfjm6jd9+rhzCjaF53/OJwv4vyVeMILndt98TBJ/jgMCf2T0gnUrgmDBB4Z/j5q95XKbajdUzX2jT+SfSHYZgOGDibEpP3hSLhdo4sTvi8HdaMT00WwaV4oHEP9vTwye7Ov9P+j1GYTkOxWbuEazCbY4bXVYplOEXhAzCXCB853i4dQX1sNFrnwgTEW9z6WYJEV/+A0O9B/cv8p1G6jGf7G/2wz7OSWpPkcq/AymWAx45drjDBopcIyWO8aZdmf282wGeK5zBzqa7HKOLq3GrCOqGfHT7PDDuDTZVYx6JVq2CxLzXwrdTZfI/HwZbjSevy6PzPTh8FgMAw4+uJMTB/d54/8z78QbFeBlKOdNeYAysyHw9FakL6U0IhKUvM0u1FbQ+yGnj7IVPNqI2zmod83mWLqWxs6sCgpUxpQ4UpSaZtJ/FFQrTxDnwmrlHSguj00wcx81PRRPQMJkvaFw8zu3MeyHF9eZhJrG1YrH8ozk8sOnP/IGiSqAkUBcvMsTr14he3/zHR4/s//rcdUPX3FnIkGg8GwF9EXG7X3Tk3AjBWDFbcbMIyrAugwoytMymxMM8nLQ0l55hBjxdsuMemhCaTWDqz7WK7BZQCrRkdg6aTiCJS8QUV7SaqVwnNbj0EWuxIsMJCHYXxQyiQlmyQpdoBJ6C3w3OcvMB8QrZCOmOIkla7AcmJTLADAAe1BkiJL4edWXmE+hfg8m4dUIrxOXJdzziRqg8FgGHD0pxRXtKPhbNj+tAmrWrsxdqMSe2EH3rragoHsSRae1YASVRRWBCcS9fQ0i0hZWWHSg8sx6dG3IUk+9ANkHmd2wMhHw3bF8haLSHB1WIqNLWHVa0xCpzbveBb6FQpgLmgefJppj23o9orAb+w2YcX1feybuGZYa2lCTXkM2tm3i+F11+lShKAvB3W7HdFmAajw55man30bO3Ci+fAKKZxhhxKtQpwYZmp5ocjGSlVfWgkEAR6YHlYVcSPwYoWmma33wErP4BCOw6oyTXiIdCN7vxYOFqLoVuXjWjR3IEMlmVs4hn1/wo6LKx9kJ39ijpkNahvscu3Ab+Jy4fdLbLG1uRZhIYsk6ta3bqJwgMFgMBjeWvTHmdiMqL0UliA7I5C1jbK7JcOS3A6kTM3tZ+E5NGStAdtFL8G6hENhqWC5zm77yWlWEm1zjZViohJ6ZBs69rK0kGC4XQLWQmx12DyMjDKpsFBiUmFmgiULRb/JamtW3h7urw01pSs/Bm0kwLQgSW2oPbgGa5eYYWaI1oVwAYzOIdZXFIZTDn81vJ7WKiZRGwwGw55FXyRqdaQIuAVzhxhfchFKIy1Q3NZD5x+VlJsN9glpEdy1KdZfbCPsYGlBu+jmNiux1ZlgtucMTKAp56GTCIYFxsbCtuxqubd2dg8lqg5MtBI0PRdvY5pB7CKw2y8wKb5JHJOSXIZpwNk0+8Y7ju3/xhIMTgBadQqu4Qb1sxwPP7P9J9f/WX8O6rhXezo8CTS2tLXJ1NDGCpioWbbAO88y1bI5xxZbM8PGOjrJLq+tSPhwpVEV8ZfZRvDH2bvl0zCedZJdmjTDMhYPHxDxBJv/WoUdStub7HCIJthYq0W21mmsemYpfJEUhtn3je+wNZxnNTLUGIHzeh8TcpowHyD7Qvi5lQkY9bUGb9YE2ItdpspMHwaDwTDg6I9ELUkgm7B+kancgg7A2kJYuh2ZYGxnhRyTMlVln7BYZpLS6BBTQ1FIFTR9NIaZtBeDaj41BxSXWXx0apw5dkiscjbHpP3Rr7D5qv04czpTdF5gmpzuZc8tJMNO/TgMRYtCp97m/UxrofOahBnMlCelcjd4Loym7IwwE0k8Ff4mLnb9fWgStcFgMAw4+iNRe8mDDKvYPiY9dsvYuRYJYH9sAoejJGmd2Z7mjzHmscXTU+yxlE8BZIl1muy7dUZhRhfkXEnHWX/qIkG8EcTOh6Xg8QcZO+GlH2Xz34aZjocXVlG7pXshe94Ok/hTk2HpsQU5V9pw/pMvM7t9vc2SwBxgnpOk1DOsv9pd4W9CE89Uh99uJ6zt+S77tS8HdSTWUW4q/HEJYYwkCVKxkgiMNvS+wzWpZIypedESTEmG0QbEKRZJsYOwXmfRF5S+slRnh1wkCetSArItSUrdETZrrRaYuS0CF0AMEkZVmuwbU6Ky/PPM2VV7ILyeovCZNBu2lYJZvRswB6HK5szPsPXeLQPwVYyOMRPp9ka4QIokpRfCjvhIF5peM30YDAbDgKMvErVzXnGQTUiIhSQp8QOmSlWnwzdq+gALMatAcpz1Enu3DgnPkRSFWZi1QljKjKWZtN+GlJ7zC+uo3eJK7/gPJCmegSGQhPp1k6nHHjpi5w+voXaX1xjHDCXlah1m60RrYdOMh3HPlK8kdYTtsdoy2zupGWb6qK5CUwowQ6Sm2d6Z+h5qprWpsAbUzdxiErXBYDAMOPoiUXc6EZVAOBpNyKgcYyFV8QthKbP5EixhNMakjNJLTHr0kPy8A52dSWAbrQMifUkaeYZJ1MtDkG4UlkQidJOS1Bxj3yQ3GZa8qqMwkxA6k2sttoWwfbfO+otUWYeRubDD3kNnPc2upYlsNVh0gZDuS1IV+mQ84Bghpe4kaet9ULMpgv66vFd/SJk6Ti2QWjk8wqI+2kk2UbV4eHOl72SUqa0LLJ61NQ4jJuCl1CYTKklDoD9I1dlg568STzOnTvvtTPVtXIJZfZAHu1oNXxBtmo4OD8KNLfZNaH3AGjwg6tDZGQXDoLUr6eFVWmHfJLYNndOgFqYkxVfZ+zUnYVQSAdzX0SwwpXQxLZnpw2AwGAYcQYnaOfdbkn5C0qr3/gTpNFJzyj4XNn20f4RJGSOQO+IKyGBsQzWPZvVFdphSklpjz63CEKM2iLeOQW4GivIB5mCZyTFNaXmUmUiylOSJcL1QE8QUzDiDzr9KjY01CbXH+jbTRhq53slijWJvTVrtWSbJpyk/y352TkSXw2dTKQv5T3LQ0Q2c/92yjcks/itJH0FvYzAYDIaeIyiWee+/5pw7eCOd+kxH9fvDAeNZaGejDpt2Nizd4IwjmDTXycCaiXex2374myzEqP3BsGOn6JjU1d7HxjA2ypIAonBeI3HIKAeZ7CLAlt2BIaG5UaYVELu4JKWhQ4x+Oz8EHXHnw5Kcg2sTa5mQE2QG1vRchTS8w8NszrYBXauHDlHqe6Lf7nromV7knHvUOfekc+7JdoF5hw0Gg8EQRs+iPrz3j0l6TJJSR/aj64Pa7SLw1nIgNbQGieM9ZOyLj0BJGbK2bR5hHA5jIO03Oc0Y1spwHqp1yNEL0etICEIPQAn8qeZFk4WqlxknSGyBaS0paBuN3hNed7RIRhSm/LdhCCRJipP4/qcFRiKV8HiTM71LspKkDtEeu7xWf8LzvNQGA/CwYsTwHKvpVyeaNKRCjBVhUYMMpDmNMOdEJ88cJ0VApk/pRlugvqUkRWHI2tYsW+SpHHMmkSxMCVLYwlC0wlmWSRibhmYDWB+S8qlQfo6RTPgyvHx+AvVFEcnCSu8wZ4Bma9ZhtXIHqjxVCkxgiqfZWLP58DqJRIzm1GAwGPYsSHje5yS9T9KEc25R0t/z3v9f3X9HioEg9QZUpShik+Fbi5KLd2aZpJSCN2qNhI5JigwxaTQGKELL27D4QRZyPcCQNQ/NAfVFyGQ3zqTgLKhCv7LJMlMdlJQpG6OH1c8FtUzBdUIoZ4dmWIJS9UWmZTRh2bGNInR2w28cLUEzxGT428WTbL6a20zyboL92s0sR6I+PonexGAwGAx9Qd9KcRGS+clZZnsuVZmNslkNS3KxdeiYmoBViGHYTXyCOc460LHTzZ71KmiIWWmJSbY0GSMDy4ltnxpH7Tw00BVq4XXSgkTvlCnOl+EWog5xmCySgHOxtBPWIDJAE5GkAtwT1OnYgMVoqQM4dZA5YmulsNaSgBK1H2LzRc7DbmuuPwd1PaLO6fDmL9/BVCRMytIOq/qtHFtEMVDjTOJVrWl1lA6NywQYhhmdJbGDugpJnmaGmSq9OQcdcbDScwmQ7VBnsq/AAx0erB7uNJ9j66kBiz2Q1dSARFCZUSZsVKDJLTXETFptMq830I5cwvSb0Oig8dFwyPKSORMNBoNh76I/4XkJr9Zc+LZsbrCwMI0zVdoBB6aPQNMCDAmikg0FlrzBTd6Cmkh8lEk2xLQk8TJWaVgQoDwEpVvw2MlpZm5bP89KLA0vsP5ouF8C1DiUpPE8JNNvhrf4NuQNaW6xdvExpilRqZUWmKCalwd0qM0kZFmExTl2QIx3Nx4ik6gNBoNhwNG3UlyxeFi6hQFLOHMqA8j0azUoAb/CpAdHk/WgVEAZxYhTbKXIpDgHw6m6EZtfiyIsbkvDBx0Mu8pcDk9G5UH2fT3kIYlBzStzkGWJlpaZv2CVhqyBsDAP95cjnMrixQ+SGcgUSLOJR6C/CMxZGiZjVWBpt0gZJAC+2VXIvYeH6zY7NFsNNlGxw2EnFqRkUrQOve+MV0blg9CJuQgzGJPhBF/+IAAADBBJREFUk5/WaXRFtgyi0HG2lmKVCGgEToRZSFQ9ETYbeJhx1o1y8lqsr8KKQb3zEUuSYs8zQaKVDj/Yj8MDDtYRpXunGYFSDpStcE3HpfBzmydgjgc1VaZBf10CCcz0YTAYDAOO/oTntSLyG2HJkN68VKIqbwJVGt66LgPjaCM0PAuW4oLPHXolfMdGWqyvwmHUTC1Ywih1mpk02in4TWC7Tjkseg29xJZ8ZRpKZxusv/IcM5FkF5nsVDoMDYdgebouKve1GDoDi19MQfWBhkrCPZvYZBJ/E5SxS/0ZzJqEhT4QHXKX1zKJ2mAwGAYc/XEmtljpqSiz16t0CNrQquEbNXuR3brlA9CmDB1dVJLPne/d3UklZWo/pWXHHBT2mtA2SrP6Rp4O2x4nnmXhb6f/CjOMNiDbHS3F5qBfl86ZAw5gD53JzSwdK5OUG6OomTow09GTSr6S9n81vO7W7oGGcbpdyTd+02lOI1IzD7J/WKFv7gEEFKGVfdDcUoNedVjQ2NFqJlDlzlwJjyN3kY21cARGOFRghhg030Thwd+G3vzSAeA4i0CiKhiT1IE7KMnuB9VHaKo5jPMFc9bKQDrcHHu3Jqxq7ygnG9z/9QnW4dI7w5MWh7VPqFDiQPRNt3Ga6cNgMBgGHH2RqKNNKXcpfA1SSS4Oq2k3U+Ebdegcu5sqs1B6YD4HjPQKDG0CUgt1wtFqNq5EQxZZuyh0EpfaUPLOhddTdZLNP3VMdWCEWX0MOonPsP62jrF26dXwXFSmWV9tEBIqCUvAB/+ILYCzj0BeE0qQBtbx8FlI/QuLH5TTYAxdzFQmURsMBsOAo382aiBpRmvs6h19nj13LRseTp1RONyA/QxKrXV4J0JpZOyF8AsuPwR5TWCYpINJIJSWtLgA7fFLMCMS2HdJaJYkxcqwZBvUMupTbEGV5piERt8vWg2Pl/seUDPs6Fw5CZOP2kzzjsCELECyqY27oI8KEkAm18Lz2s3e3Z84as/UWuodptEL0XL448ZZJq/Sa6wdNX200uxT18bZKm/kwxOfP8/6Ku+HDlYYpUM2giRFoCOmNtG74sa5i2wDtugZAs1LFJllWMj1IOuvDgrNJuCeSF5g7UrzUNqAFzq9lKiQg004ADHodLxZmOnDYDAYBhz9cSY2pPylsKpXmYDSDQwfigFTSqLYWydhbZy9W3aR9deB4ZstoIbSrMnMMntmA9JaUIkluQHrV0KHXScONCo4/600e7ckdJzKsbVOtcdID8uNpjbYN9m+nfWXWYIPhp+OStQ03Bfxn8B9SM18yJRqmYkGg8Gwd9EXibodl4rEKYKzq1g7Ymujffk4lEZX2CCmvrmB2q2+k9URbAAG0/oYlArZq2GmwDa079LMVCpOkNBG5yFvCGQeHH6eObp2POWhQM3kYKIVKQHWgj4F6nSsTbL+EqzmAj4naPLRvm+EP97l9zKROsGqzmn4HHhmxdjzDAaDYc+iP+F5MRbgP/1dJt6u381ek9iLWikYwgOlQjnI0fwgk5Tb8P1IxASVWOlYx15kYlxtClZ6h89Nr0K/AvBlVCfh94USawdSvbQZzTiOIqD9kRRnEhki8QSl5jCbL1iEHIfxQqVFKydBFXIo7ecuszOsuD882HYXLb4vB3WkJaXWAccAdNg0IccAok2EsaDUqRcv9TYEDocYAdWcOutobPnavZDoH14Q9BuX4LdrgDjqNAwJTV+BFLEL7HSYeJbFIm4fYVuyPgpj0JfD48VCCTVVwkuO6vONUWZeilahYx84O8uQE2j9bljPEwyh234w04fBYDAMOPrjTExKxcPhK6R4CIZnAQ4PSWplwb0DCfwF6SZrkO2OgrKi+WT4BXdg8gwlcHc0u7LHIGOVhKhkyyk2hvIceyR9t/IC/Ha9jLuTVAX2wBZkO+yk2VhJ4pEkuQbc/0NMG2nn2DfezIfb+QgbK+XJIWvTd9GSTaI2GAyGAQeSqJ1zH5H0aUlRSb/pvf+fuv5CR4oAB0BrFOYQQ4mPEMxHoB2rnYeSDZTQo1tMeYlAKaMNJCU/yrw/bhMWD4bhWdQ2TstTUZ3FNUB5Msgz3s7S4qasGRaJoPJApVbUFxXi6bvBNUxY7CSpDkMWRTifxYp9ULu9h/Nws6W4gqeHcy4q6V9K+pCkRUnfc859wXt/XaqkaF0afiX8XlvHmSF+6knUTJvHwh/tyGfXUV8vPcqiNOIFtgMX/pgFXC6/k7GupzbDbdYeZN66uSdg3TcYW75xjM1rHlazaaVZO0JNWZ6Fh8MIu1jTa5RMHx6s8DCkPClTT4UrFsRfuIj6Ov9f3YHa0WIa8//nC6jd6b9zJ2pHI3BSQJBoJ9h85RbZhJXmwuupm5BGdsA7JJ323p/13jck/Z6kj6O3MxgMBsNNw/lAppZz7hFJH/He/8Lu//+spAe997/8mnaPSnp093/vkPRS71/3TceEJCaCDz5ulbHcKuOQbCyDirdqLAve+9fN6yT63evJ43/udPfePybpsRt8sYGGc+5J7/3Jt/o9eoFbZSy3yjgkG8ugYhDHQkwfi5Lmr/n/OUlX+vM6BoPBYHgtyEH9PUm3OecOOecSkn5G0hf6+1oGg8FgeBVB04f3vuWc+2VJj+tqeN5vee+f6/ubDQZuJVPOrTKWW2Ucko1lUDFwYwk6Ew0Gg8Hw1sIyEw0Gg2HAYQe1wWAwDDhuuYPaOfcR59xLzrnTzrlfe52fO+fcv9j9+TPOuftDv+ucG3POfdk598ruf0ev+dmv77Z/yTn34Wv+/X90zl1yzpX28liccxnn3B875150zj3nnOtOHzCg49j99y86536wO47/fTfrdk+O5Zqff8E5d+pGxzFIY3HOfWX3357e/TO1h8eScM495px7eXfP/OUbHcvrwnt/y/zRVWfnGUmHJSUk/UDSsde0+Zik/6ir8eEPSfpO6Hcl/SNJv7b791+T9A93/35st11S0qHd34/u/uwhSbOSSnt5LLrK4P3+3TYJSV+X9NG9No7dnw3t/tdJ+kNJP7MX5+SaZ/1nkn5X0qm9ur52f/YVSSdvkX3/P0j6jd2/RyRN9OJsu9UkapLu/nFJn/FX8W1JI8652cDvflzS7+z+/Xck/aVr/v33vPd17/05Sad3+5H3/tvee1qPeWDH4r2veO//390xNSR9X1dj6ffUOHbf/9WqmjFd3ZQ36kkfmLE453KS/rqk37jBMQzcWHqAQRrLz0v6B5Lkve9473uS4XirHdT7JV265v8Xd/+NtOn2u9OvHrq7/31VNSPPe6MYuLE450Yk/aSkJ/bqOJxzj0talVSU9Ac3MI5BG8vfl/RPJVVucAyh9yRt+rG+fnvX7PHfOgfr2w3YWHb3hyT9fefc951zv++cm77BsbwubrWDmqS7X68NSpV/A897oxiosTjnYpI+J+lfeO/PBvrC/Qba9Hwc3vsP66pJKinpA4G+bqjvQJuejcU5d6+ko977zwd+/4b7hm16PS9/1Xt/l6R37/752UBfN9J3qE0vxxLTVW3zT73390v6lqR/EugL4VY7qEm6+/XadPvdlV01Sbv/Xb2B571RDNpYHpP0ivf+n+/xcch7X9PV7NobZYEclLG8U9LbnXPnJX1D0u3Oua/s0bHIe395979FXbW536hJZFDGsqGrGs6rF+jvS7pfvUAvDN2D8kdXb7Szumrgf9UxcPw1bX5cP+xU+G7odyX9Y/2wU+Ef7f79uH7YqXBW1zh7dtu8UWfiwIxFV+2gfygpslfHISknafaafv+NpF/ei2N5zfMO6o05EwdiLLt9Tey2ieuqOeoX9+JYdn/2e5I+sPv3n5P0+zd7rnnvb62DevfjfEzSy7rqif27u//2i69O/u5E/cvdnz+ra7zNr/e7u/8+rqt22Vd2/zt2zc/+7m77l3RNNISueowXdZUKflHSf78Xx6Kr0oKX9IKkp3f//MIeHMe0rvLWPCPpOUn/i6TYXpyT17zPQb2Bg3pQxiIpK+mpa+bl03rNZbRXxrL77wuSvrY7nickHXgjc/PaP5ZCbjAYDAOOW81GbTAYDLcc7KA2GAyGAYcd1AaDwTDgsIPaYDAYBhx2UBsMBsOAww5qg8FgGHDYQW0wGAwDjv8PZ2/oNTFVYDQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plt.specgram(\n",
    "    val[0],\n",
    "    NFFT=256,\n",
    "    Fs=122.68e6,\n",
    "    noverlap=18,\n",
    ") \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "cb356930",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(251, 251)\n"
     ]
    }
   ],
   "source": [
    "print(zxx.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "5f3e9b91",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_fft_and_scaler(data, start=5192, end=8192):\n",
    "    data = np.fft.fft(data)\n",
    "    data = np.abs(data)\n",
    "    data = data/np.expand_dims(data.max(axis=1), axis=1)\n",
    "    return data[:,start:end]\n",
    "\n",
    "def get_stft(data, nfft=126, x=126, y=126, windows=30, oneside=False, overlap=6, dtype='complex64'):\n",
    "    fs = 122.68e6\n",
    "    result = np.zeros((data.shape[0], x, y), dtype=dtype)\n",
    "    for idx, i in enumerate(tqdm(data)):\n",
    "        f, t, zxx = signal.stft(\n",
    "        i, \n",
    "        fs=fs,\n",
    "        nfft=nfft,\n",
    "        window=signal.get_window('hann', windows),\n",
    "        noverlap=overlap,\n",
    "        nperseg=windows,\n",
    "        return_onesided=oneside) # 设置双边谱\n",
    "        result[idx, :] = zxx[:, :].astype(dtype)\n",
    "    return result\n",
    "    \n",
    "def get_image(stft, dtype='float32'):\n",
    "    real = np.zeros_like(stft, dtype=dtype)\n",
    "    imag = np.zeros_like(stft, dtype=dtype)\n",
    "    angel = np.zeros_like(stft, dtype=dtype)\n",
    "    for idx, i in enumerate(tqdm(stft)):\n",
    "        temp_imag = i.imag\n",
    "        real[idx, :] = i.real.astype(dtype)\n",
    "        imag[idx, :] = i.imag.astype(dtype)\n",
    "        angel[idx, :] = np.arctan(i.real/i.imag).astype(dtype)\n",
    "    length = len(real)\n",
    "    image = np.stack([real, imag, angel], axis=1) # 这里只要将维度设置为1即可\n",
    "    del real, imag, angel; gc.collect()\n",
    "    return image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "d19a69b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▍        | 10527/70209 [00:05<00:28, 2101.22it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-41-156ebae72de4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mstft\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_stft\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_sp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mimage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_image\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstft\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mdel\u001b[0m \u001b[0mstft\u001b[0m\u001b[0;34m;\u001b[0m \u001b[0mgc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-40-2a811be31d04>\u001b[0m in \u001b[0;36mget_stft\u001b[0;34m(data, nfft, x, y, windows, oneside, overlap, dtype)\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m         f, t, zxx = signal.stft(\n\u001b[0m\u001b[1;32m     12\u001b[0m         \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0mfs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/scipy/signal/spectral.py\u001b[0m in \u001b[0;36mstft\u001b[0;34m(x, fs, window, nperseg, noverlap, nfft, detrend, return_onesided, boundary, padded, axis)\u001b[0m\n\u001b[1;32m   1167\u001b[0m     \"\"\"\n\u001b[1;32m   1168\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1169\u001b[0;31m     freqs, time, Zxx = _spectral_helper(x, x, fs, window, nperseg, noverlap,\n\u001b[0m\u001b[1;32m   1170\u001b[0m                                         \u001b[0mnfft\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdetrend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_onesided\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1171\u001b[0m                                         \u001b[0mscaling\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'spectrum'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/scipy/signal/spectral.py\u001b[0m in \u001b[0;36m_spectral_helper\u001b[0;34m(x, y, fs, window, nperseg, noverlap, nfft, detrend, return_onesided, scaling, axis, mode, boundary, padded)\u001b[0m\n\u001b[1;32m   1828\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1829\u001b[0m     \u001b[0;31m# Perform the windowed FFTs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1830\u001b[0;31m     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_fft_helper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwin\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdetrend_func\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnperseg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnoverlap\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnfft\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msides\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1831\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1832\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msame_data\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/scipy/signal/spectral.py\u001b[0m in \u001b[0;36m_fft_helper\u001b[0;34m(x, win, detrend_func, nperseg, noverlap, nfft, sides)\u001b[0m\n\u001b[1;32m   1913\u001b[0m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreal\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1914\u001b[0m         \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msp_fft\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrfft\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1915\u001b[0;31m     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnfft\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1916\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1917\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "stft = get_stft(train_sp)\n",
    "\n",
    "image = get_image(stft)\n",
    "del stft; gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "5a546a31",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = models.resnet50()\n",
    "\n",
    "# 这里改成自适应，因为主要是Maxpooling层减小图片尺寸，只要改成自适应就好了\n",
    "for name, layer in model.named_modules():\n",
    "    if isinstance(layer, nn.MaxPool2d):\n",
    "        model.maxpool = nn.AdaptiveAvgPool2d((7, 7))    \n",
    "\n",
    "n_class = 10\n",
    "numFit = model.fc.in_features\n",
    "model.fc = nn.Linear(numFit, n_class) # 直接修改最后一层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "59e27253",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyDataset(Dataset):\n",
    "    def __init__(self, file_list, label_list):\n",
    "        self.file_list = file_list\n",
    "        self.label_list = label_list\n",
    "        \n",
    "    def __len__(self):\n",
    "        self.filelength = len(self.file_list)\n",
    "        return self.filelength\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        img = torch.tensor(self.file_list[idx])\n",
    "        label = self.label_list[idx]\n",
    "        \n",
    "        return img, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ef419d25",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7e075300",
   "metadata": {},
   "outputs": [],
   "source": [
    "image_labels = torch.tensor(train_label)\n",
    "x_train, x_test, y_train, y_test = train_test_split(image, image_labels, test_size=0.3, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "58010439",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "train_data = MyDataset(x_train, y_train)\n",
    "test_data = MyDataset(x_test, y_test)\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "device_count = torch.cuda.device_count()\n",
    "USE_CUDA = torch.cuda.is_available()\n",
    "device = torch.device(\"cuda:0\" if USE_CUDA else \"cpu\")\n",
    "\n",
    "if device_count > 1:\n",
    "    model = nn.DataParallel(model,device_ids=range(device_count)) # multi-GPU\n",
    "    model.to(device)\n",
    "\n",
    "else:\n",
    "    model = model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "418c04ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "epochs = 10\n",
    "lr = 0.001\n",
    "gamma = 0.9\n",
    "step_size = 1\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=gamma) # 学习方式\n",
    "\n",
    "# 训练模型\n",
    "train_acc, test_acc = [], []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "c81407e3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 768/768 [02:45<00:00,  4.64it/s]\n",
      "100%|██████████| 330/330 [00:43<00:00,  7.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 0, train loss:2.3577, train acc:0.2307\n",
      "test loss:2.2446, test acc:0.2646\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 768/768 [02:34<00:00,  4.96it/s]\n",
      "100%|██████████| 330/330 [00:42<00:00,  7.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 1, train loss:2.1968, train acc:0.2822\n",
      "test loss:2.1744, test acc:0.3009\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 768/768 [02:27<00:00,  5.22it/s]\n",
      "100%|██████████| 330/330 [00:41<00:00,  7.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 2, train loss:1.8964, train acc:0.3908\n",
      "test loss:1.6364, test acc:0.4719\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 768/768 [02:30<00:00,  5.09it/s]\n",
      "100%|██████████| 330/330 [00:41<00:00,  7.88it/s]\n",
      "  0%|          | 0/768 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 3, train loss:1.5857, train acc:0.4906\n",
      "test loss:1.4556, test acc:0.5180\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 768/768 [02:30<00:00,  5.09it/s]\n",
      "100%|██████████| 330/330 [00:39<00:00,  8.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 4, train loss:1.4016, train acc:0.5430\n",
      "test loss:1.3863, test acc:0.5527\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 768/768 [02:32<00:00,  5.03it/s]\n",
      "100%|██████████| 330/330 [00:41<00:00,  8.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 5, train loss:1.3602, train acc:0.5495\n",
      "test loss:1.2562, test acc:0.5784\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 768/768 [02:27<00:00,  5.21it/s]\n",
      "100%|██████████| 330/330 [00:40<00:00,  8.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 6, train loss:1.1469, train acc:0.6177\n",
      "test loss:1.1327, test acc:0.6295\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 768/768 [02:26<00:00,  5.25it/s]\n",
      "100%|██████████| 330/330 [00:42<00:00,  7.85it/s]\n",
      "  0%|          | 0/768 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 7, train loss:1.0187, train acc:0.6570\n",
      "test loss:0.9908, test acc:0.6660\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 768/768 [02:27<00:00,  5.22it/s]\n",
      "100%|██████████| 330/330 [00:38<00:00,  8.47it/s]\n",
      "  0%|          | 0/768 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 8, train loss:0.9575, train acc:0.6722\n",
      "test loss:0.9815, test acc:0.6689\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 768/768 [02:29<00:00,  5.12it/s]\n",
      "100%|██████████| 330/330 [00:39<00:00,  8.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 9, train loss:0.9131, train acc:0.6849\n",
      "test loss:0.9100, test acc:0.6931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(epochs):\n",
    "    epoch_loss = 0\n",
    "    epoch_accuracy = 0\n",
    "    for data, label in tqdm(train_loader):\n",
    "        data = data.cuda()\n",
    "        label = label.cuda()\n",
    "        output = model(data)\n",
    "    \n",
    "        loss = criterion(output, label)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        acc = (output.argmax(dim=1) == label).float().mean()\n",
    "        epoch_accuracy += acc / len(train_loader)\n",
    "        epoch_loss += loss / len(train_loader)\n",
    "        \n",
    "        data.cpu()\n",
    "        label.cpu()\n",
    "        \n",
    "    with torch.no_grad():\n",
    "        epoch_test_accuracy = 0\n",
    "        epoch_test_loss = 0\n",
    "        for data, label in tqdm(test_loader):\n",
    "            data = data.cuda()\n",
    "            label = label.cuda()\n",
    "            \n",
    "            test_output = model(data)\n",
    "            test_loss = criterion(test_output, label)\n",
    "            \n",
    "            acc = (test_output.argmax(dim=1) == label).float().mean()\n",
    "            epoch_test_accuracy += acc / len(test_loader)\n",
    "            epoch_test_loss += test_loss / len(test_loader)\n",
    "            data.cpu()\n",
    "            label.cpu()\n",
    "    scheduler.step()        \n",
    "    print(f'EPOCH:{epoch:2}, train loss:{epoch_loss:.4f}, train acc:{epoch_accuracy:.4f}')\n",
    "    print(f'test loss:{epoch_test_loss:.4f}, test acc:{epoch_test_accuracy:.4f}')\n",
    "\n",
    "    train_acc.append(epoch_accuracy)\n",
    "    test_acc.append(epoch_test_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "51b0f2a4",
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "CUDA out of memory. Tried to allocate 79.73 GiB (GPU 0; 22.38 GiB total capacity; 15.24 GiB already allocated; 6.24 GiB free; 15.42 GiB reserved in total by PyTorch)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m--------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-144-ce87397f2795>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFloatTensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    887\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 889\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    890\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    891\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torchvision/models/resnet.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    247\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    248\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 249\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    250\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    251\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torchvision/models/resnet.py\u001b[0m in \u001b[0;36m_forward_impl\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    230\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_forward_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    231\u001b[0m         \u001b[0;31m# See note [TorchScript super()]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 232\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    233\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbn1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    887\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 889\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    890\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    891\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/nn/modules/conv.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    397\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    398\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 399\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_conv_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    400\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    401\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mConv3d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_ConvNd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/nn/modules/conv.py\u001b[0m in \u001b[0;36m_conv_forward\u001b[0;34m(self, input, weight, bias)\u001b[0m\n\u001b[1;32m    393\u001b[0m                             \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstride\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    394\u001b[0m                             _pair(0), self.dilation, self.groups)\n\u001b[0;32m--> 395\u001b[0;31m         return F.conv2d(input, weight, bias, self.stride,\n\u001b[0m\u001b[1;32m    396\u001b[0m                         self.padding, self.dilation, self.groups)\n\u001b[1;32m    397\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: CUDA out of memory. Tried to allocate 79.73 GiB (GPU 0; 22.38 GiB total capacity; 15.24 GiB already allocated; 6.24 GiB free; 15.42 GiB reserved in total by PyTorch)"
     ]
    }
   ],
   "source": [
    "# 直接将所有的放进去容易显存溢出，所以要分批次放入\n",
    "model(torch.FloatTensor(x_test).to(device))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "70e449a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_save_name = 'resnet_best_model.point'\n",
    "torch.save(model.state_dict(), model_save_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "ccf57ebc",
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = np.zeros(x_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "id": "906811b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21063,)"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "85b23be7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21063, 3, 251, 251)"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "0b59fd57",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., ..., 0., 0., 0.])"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "ece02c28",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test_t = torch.FloatTensor(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "2bc05270",
   "metadata": {},
   "outputs": [],
   "source": [
    "res = torch.chunk(x_test_t, chunks=128, dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "5fabb529",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([165, 3, 251, 251])"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res[-2].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "eda74aa8",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch = 128\n",
    "res = np.array_split(x_test, batch, axis=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "da917aac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(165, 3, 251, 251)"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "40cf5fc0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(164, 3, 251, 251)"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res[-1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "727481a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(164, 3, 251, 251)"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res[-2].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "id": "bb8d9f5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_submmit(model, test, batch=128, dim=0):\n",
    "    # 初始化结果提交向量\n",
    "    # test为numpy类型的数据\n",
    "    preds = np.zeros(test.shape[0])\n",
    "    res = torch.FloatTensor(test)\n",
    "    res = torch.chunk(res, chunks=batch, dim=dim)\n",
    "    start = 0\n",
    "    end = 0\n",
    "    for chunk_data in tqdm(res):\n",
    "        chunk_data = chunk_data.to(device)\n",
    "        output = model(chunk_data).argmax(dim=1).detach().cpu().numpy()\n",
    "        end += output.shape[0]\n",
    "        preds[start:end] = output\n",
    "        start = end\n",
    "        \n",
    "    return preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "id": "d3c38e09",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 128/128 [00:11<00:00, 11.57it/s]\n"
     ]
    }
   ],
   "source": [
    "preds = get_submmit(model, x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "id": "9ef4adf6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6973840383611072"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(preds == y_test.numpy()).sum()/preds.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "1b93fa2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([6, 1, 9, ..., 1, 1, 3])"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "id": "cabb83cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('preds.npy', preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ec54e4c",
   "metadata": {},
   "source": [
    "## Resnet50 or 18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ecfb2542",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import Counter\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision\n",
    "from tqdm import tqdm\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "from torch.utils.data import TensorDataset, DataLoader, Dataset\n",
    "from scipy import signal\n",
    "import gc\n",
    "from torchvision import models\n",
    "\n",
    "# 固定随机数种子，确保实验的可重复性\n",
    "def set_seed(seed=42):\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "\n",
    "set_seed(42)\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "0c67d80f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stage 1: load data\n"
     ]
    }
   ],
   "source": [
    "train_path = './train'\n",
    "val_path = './val'\n",
    "test_path = './test'\n",
    "\n",
    "# 读取训练集，测试集和验证集\n",
    "print('Stage 1: load data')\n",
    "\n",
    "train = np.load(train_path + '/' + '10type_sort_train_data_8192.npy')\n",
    "test = np.load(test_path + '/' + '10type_sort_test_data_8192.npy')\n",
    "val = np.load(val_path + '/' + '10type_sort_eval_data_8192.npy')\n",
    "\n",
    "# 读取训练集和验证集的标签，测试集是没有标签的，需要你使用模型进行分类，并将结果进行提交\n",
    "train_label = np.load(train_path + '/' + '10type_sort_train_label_8192.npy')\n",
    "val_label = np.load(val_path + '/' + '10type_sort_eval_label_8192.npy')\n",
    "\n",
    "def get_fft_and_scaler(data, start=5192, end=8192):\n",
    "    data = np.fft.fft(data)\n",
    "    data = np.abs(data)\n",
    "    data = data/np.expand_dims(data.max(axis=1), axis=1)\n",
    "    return data[:,start:end]\n",
    "\n",
    "def get_stft(\n",
    "        data,\n",
    "        nfft=126,\n",
    "        x=126,\n",
    "        y=126,\n",
    "        windows=30,\n",
    "        oneside=False,\n",
    "        overlap=6,\n",
    "        dtype='complex64'):\n",
    "    fs = 122.68e6\n",
    "    result = np.zeros((data.shape[0], x, y), dtype=dtype)\n",
    "    for idx, i in enumerate(tqdm(data)):\n",
    "        f, t, zxx = signal.stft(\n",
    "        i,\n",
    "        fs=fs,\n",
    "        nfft=nfft,\n",
    "        window=signal.get_window('hann', windows),\n",
    "        noverlap=overlap,\n",
    "        nperseg=windows,\n",
    "        return_onesided=oneside) # 设置双边谱\n",
    "        result[idx, :] = zxx[:, :].astype(dtype)\n",
    "    return result\n",
    "\n",
    "def get_image(stft, dtype='float32'):\n",
    "    real = np.zeros_like(stft, dtype=dtype)\n",
    "    imag = np.zeros_like(stft, dtype=dtype)\n",
    "    angel = np.zeros_like(stft, dtype=dtype)\n",
    "    for idx, i in enumerate(tqdm(stft)):\n",
    "        temp_imag = i.imag\n",
    "        real[idx, :] = i.real.astype(dtype)\n",
    "        imag[idx, :] = i.imag.astype(dtype)\n",
    "        angel[idx, :] = np.arctan(i.real/i.imag).astype(dtype)\n",
    "    length = len(real)\n",
    "    image = np.stack([real, imag, angel], axis=1) # 这里只要将维度设置为1即可\n",
    "    del real, imag, angel; gc.collect()\n",
    "    return image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "99a9f4ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stage 2: data split\n"
     ]
    }
   ],
   "source": [
    "# 获取切分的数据\n",
    "print(\"stage 2: data split\")\n",
    "train_sp = get_fft_and_scaler(train)\n",
    "test_sp = get_fft_and_scaler(test)\n",
    "val_sp = get_fft_and_scaler(val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "e2d8ed7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 213/70209 [00:00<00:32, 2124.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stage 3: data transform(fft series --> image)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 70209/70209 [00:32<00:00, 2134.81it/s]\n",
      "  0%|          | 0/70209 [00:00<?, ?it/s]<ipython-input-46-0eb06601b146>:53: RuntimeWarning: divide by zero encountered in true_divide\n",
      "  angel[idx, :] = np.arctan(i.real/i.imag).astype(dtype)\n",
      "100%|██████████| 70209/70209 [00:24<00:00, 2810.00it/s]\n",
      "100%|██████████| 23403/23403 [00:11<00:00, 2125.69it/s]\n",
      "100%|██████████| 23403/23403 [00:08<00:00, 2809.57it/s]\n",
      "100%|██████████| 23403/23403 [00:11<00:00, 2120.64it/s]\n",
      "100%|██████████| 23403/23403 [00:08<00:00, 2811.14it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将数据转换成图片\n",
    "print('Stage 3: data transform(fft series --> image)')\n",
    "\n",
    "train_stft = get_stft(train_sp)\n",
    "del train_sp; gc.collect()\n",
    "train_image = get_image(train_stft)\n",
    "del train_stft; gc.collect()\n",
    "\n",
    "val_stft = get_stft(val_sp)\n",
    "del val_sp; gc.collect()\n",
    "val_image = get_image(val_stft)\n",
    "del val_stft; gc.collect()\n",
    "\n",
    "test_stft = get_stft(test_sp)\n",
    "del test_sp; gc.collect()\n",
    "test_image = get_image(test_stft)\n",
    "del test_stft; gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "4d055324",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyDataset(Dataset):\n",
    "    def __init__(self, file_list, label_list):\n",
    "        self.file_list = file_list\n",
    "        self.label_list = label_list\n",
    "\n",
    "    def __len__(self):\n",
    "        self.filelength = len(self.file_list)\n",
    "        return self.filelength\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        img = torch.tensor(self.file_list[idx])\n",
    "        label = self.label_list[idx]\n",
    "\n",
    "        return img, label\n",
    "\n",
    "# 数据是10分类所以设置为10\n",
    "n_class = 10\n",
    "batch_size = 64\n",
    "epochs = 15\n",
    "lr = 0.001\n",
    "gamma = 0.9\n",
    "step_size = 1\n",
    "\n",
    "# 将训练集和验证机封装成torch的数据，方便dataloader读取数据\n",
    "# 这里直接传如numpy数据即可\n",
    "train_data = MyDataset(train_image, train_label)\n",
    "val_data = MyDataset(val_image, val_label)\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, shuffle=True, drop_last=True)\n",
    "val_loader = torch.utils.data.DataLoader(val_data, batch_size=batch_size, shuffle=True, drop_last=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "719ec55c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 设置模型\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "choose = 18\n",
    "if choose == 18:\n",
    "    model = models.resnet18().to(device)\n",
    "else:\n",
    "    model = models.resnet50().to(device)\n",
    "# 这里改成自适应，因为主要是Maxpooling层减小图片尺寸，只要改成自适应就好了\n",
    "for name, layer in model.named_modules():\n",
    "    if isinstance(layer, nn.MaxPool2d):\n",
    "        model.maxpool = nn.AdaptiveAvgPool2d((7, 7))\n",
    "\n",
    "model.fc.in_features = n_class\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size= step_size, gamma=gamma) # 学习方式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "45137e2d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 3/1097 [00:00<00:51, 21.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stage 4: train model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [00:44<00:00, 24.81it/s]\n",
      "100%|██████████| 365/365 [00:05<00:00, 62.88it/s]\n",
      "  0%|          | 3/1097 [00:00<00:45, 24.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 0, train loss:1.9216, train acc:0.3552\n",
      "test loss:1.7626, test acc:0.4175\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [00:44<00:00, 24.90it/s]\n",
      "100%|██████████| 365/365 [00:05<00:00, 62.50it/s]\n",
      "  0%|          | 3/1097 [00:00<00:45, 24.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 1, train loss:1.2629, train acc:0.5752\n",
      "test loss:1.6958, test acc:0.4687\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [00:43<00:00, 24.98it/s]\n",
      "100%|██████████| 365/365 [00:05<00:00, 62.49it/s]\n",
      "  0%|          | 0/1097 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 2, train loss:1.0348, train acc:0.6452\n",
      "test loss:1.8046, test acc:0.4489\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [00:43<00:00, 24.93it/s]\n",
      "100%|██████████| 365/365 [00:05<00:00, 62.81it/s]\n",
      "  0%|          | 0/1097 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 3, train loss:0.9138, train acc:0.6837\n",
      "test loss:1.7745, test acc:0.4690\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [00:44<00:00, 24.85it/s]\n",
      "100%|██████████| 365/365 [00:05<00:00, 62.60it/s]\n",
      "  0%|          | 0/1097 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 4, train loss:0.8426, train acc:0.7065\n",
      "test loss:1.8476, test acc:0.4825\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [00:44<00:00, 24.89it/s]\n",
      "100%|██████████| 365/365 [00:05<00:00, 61.65it/s]\n",
      "  0%|          | 0/1097 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 5, train loss:0.7959, train acc:0.7222\n",
      "test loss:1.8718, test acc:0.4770\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [00:43<00:00, 25.02it/s]\n",
      "100%|██████████| 365/365 [00:05<00:00, 63.79it/s]\n",
      "  0%|          | 3/1097 [00:00<00:45, 23.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 7, train loss:0.7104, train acc:0.7476\n",
      "test loss:1.9444, test acc:0.5027\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [00:43<00:00, 25.02it/s]\n",
      "100%|██████████| 365/365 [00:05<00:00, 63.87it/s]\n",
      "  0%|          | 0/1097 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 8, train loss:0.6753, train acc:0.7598\n",
      "test loss:1.9272, test acc:0.4926\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [00:43<00:00, 25.02it/s]\n",
      "100%|██████████| 365/365 [00:05<00:00, 63.19it/s]\n",
      "  0%|          | 0/1097 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 9, train loss:0.6382, train acc:0.7730\n",
      "test loss:2.0049, test acc:0.4987\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [00:43<00:00, 24.95it/s]\n",
      "100%|██████████| 365/365 [00:05<00:00, 62.19it/s]\n",
      "  0%|          | 0/1097 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:10, train loss:0.6091, train acc:0.7833\n",
      "test loss:1.9454, test acc:0.4880\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [00:43<00:00, 24.95it/s]\n",
      "100%|██████████| 365/365 [00:05<00:00, 63.77it/s]\n",
      "  0%|          | 0/1097 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:11, train loss:0.5807, train acc:0.7929\n",
      "test loss:2.0710, test acc:0.4916\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [00:43<00:00, 25.02it/s]\n",
      "100%|██████████| 365/365 [00:05<00:00, 63.14it/s]\n",
      "  0%|          | 3/1097 [00:00<00:45, 24.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:12, train loss:0.5508, train acc:0.8020\n",
      "test loss:2.1855, test acc:0.4819\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [00:44<00:00, 24.91it/s]\n",
      "100%|██████████| 365/365 [00:05<00:00, 61.93it/s]\n",
      "  0%|          | 0/1097 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:13, train loss:0.5220, train acc:0.8111\n",
      "test loss:2.2151, test acc:0.4869\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 798/1097 [00:32<00:12, 24.81it/s]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 训练模型\n",
    "print('Stage 4: train model')\n",
    "best_loss = float('inf')\n",
    "best_model = None\n",
    "train_acc, val_acc = [], []\n",
    "for epoch in range(epochs):\n",
    "    epoch_loss = 0\n",
    "    epoch_accuracy = 0\n",
    "    for data, label in tqdm(train_loader):\n",
    "        data = data.cuda()\n",
    "        label = label.cuda()\n",
    "        output = model(data)\n",
    "        loss = criterion(output, label)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        acc = (output.argmax(dim=1) == label).float().mean()\n",
    "        epoch_accuracy += acc / len(train_loader)\n",
    "        epoch_loss += loss / len(train_loader)\n",
    "\n",
    "        data.cpu()\n",
    "        label.cpu()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        epoch_val_accuracy = 0\n",
    "        epoch_val_loss = 0\n",
    "        for data, label in tqdm(val_loader):\n",
    "            data = data.cuda()\n",
    "            label = label.cuda()\n",
    "            val_output = model(data)\n",
    "            val_loss = criterion(val_output, label)\n",
    "            acc = (val_output.argmax(dim=1) == label).float().mean()\n",
    "            epoch_val_accuracy += acc / len(val_loader)\n",
    "            epoch_val_loss += val_loss / len(val_loader)\n",
    "\n",
    "            data.cpu()\n",
    "            label.cpu()\n",
    "\n",
    "    scheduler.step()\n",
    "    print(f'EPOCH:{epoch:2}, train loss:{epoch_loss:.4f}, train acc:{epoch_accuracy:.4f}')\n",
    "    print(f'test loss:{epoch_val_loss:.4f}, test acc:{epoch_val_accuracy:.4f}')\n",
    "\n",
    "    train_acc.append(epoch_accuracy)\n",
    "    val_acc.append(epoch_val_accuracy)\n",
    "\n",
    "    if epoch_loss <= best_loss:\n",
    "        best_loss = epoch_loss\n",
    "        best_model = model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "86cbe9f2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stage 5: model save\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/128 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stage 6: predict\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 128/128 [00:12<00:00, 10.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stage 7: save predict\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# 保存模型\n",
    "print('Stage 5: model save')\n",
    "model_save_name = 'resnet_best_model.point'\n",
    "torch.save(best_model.state_dict(), model_save_name)\n",
    "\n",
    "# 生成预测结果\n",
    "print('Stage 6: predict')\n",
    "model.load_state_dict(torch.load(model_save_name))\n",
    "\n",
    "# 直接将所有的放进去容易显存溢出，所以要分批次放入\n",
    "def get_submmit(model, test, batch=128, dim=0):\n",
    "    # 初始化结果提交向量\n",
    "    # test为numpy类型的数据\n",
    "    preds = np.zeros(test.shape[0])\n",
    "    res = torch.FloatTensor(test)\n",
    "    res = torch.chunk(res, chunks=batch, dim=dim)\n",
    "    start = 0\n",
    "    end = 0\n",
    "    for chunk_data in tqdm(res):\n",
    "        chunk_data = chunk_data.to(device)\n",
    "        output = model(chunk_data).argmax(dim=1).detach().cpu().numpy()\n",
    "        end += output.shape[0]\n",
    "        preds[start:end] = output\n",
    "        start = end\n",
    "\n",
    "    return preds\n",
    "\n",
    "submmit = get_submmit(model, test_image)\n",
    "\n",
    "# 还要保存预测结果\n",
    "print('Stage 7: save predict')\n",
    "np.save('submmit.npy', submmit)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca9c4aa9",
   "metadata": {},
   "source": [
    "# CNN + LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "183ee39e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import TensorDataset, Dataset, DataLoader\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "ec401457",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CRNN(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, num_layers, num_classes):\n",
    "        super().__init__()\n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        self.cnn = nn.Sequential(\n",
    "            nn.Conv2d(3, 64, kernel_size=(9, 9), stride=(2, 2), padding=(3, 3)),\n",
    "            nn.BatchNorm2d(64),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.AdaptiveAvgPool2d(output_size=(self.input_size, self.input_size)),\n",
    "            nn.Conv2d(64, 32, kernel_size=(6, 6), stride=(1, 1)),\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.Conv2d(32, 16, kernel_size=(3, 3), stride=(1, 1)),\n",
    "            nn.BatchNorm2d(16),\n",
    "            nn.Conv2d(16, 1, kernel_size=(1, 1), stride=(1, 1)),\n",
    "        )\n",
    "        # 官网默认的输入是seq_len, batch, input_size,这里将batch first=True放在第一个\n",
    "        # seq len 相当于一句话的长度， input size相当于每个单词的向量长度\n",
    "        self.lstm = nn.LSTM(self.input_size - 7, hidden_size, num_layers, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_size, num_classes)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.cnn(x).squeeze(dim=1)\n",
    "        #print(x.shape)\n",
    "        #assert 0\n",
    "        # 初始化两个向量的值, 因为这里是两层的lstm\n",
    "        \n",
    "        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)\n",
    "        c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)\n",
    "\n",
    "        # 前向传播，这里只需要out即可\n",
    "        #print(x.shape)\n",
    "        #assert 0\n",
    "        out, _ = self.lstm(x, (h0, c0))  # out: tensor of shape (batch_size, seq_length, hidden_size)\n",
    "\n",
    "        # 使用全连层进行分类\n",
    "        out = self.fc(out[:, -1, :])\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "ad35748e",
   "metadata": {},
   "outputs": [],
   "source": [
    "sequence_length = 1\n",
    "input_size = 35\n",
    "hidden_size = 512\n",
    "num_layers = 2\n",
    "num_classes = 10\n",
    "batch_size = 128\n",
    "num_epochs = 30\n",
    "learning_rate = 0.001\n",
    "gamma = 1\n",
    "step_size=1\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "dc3ca140",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CRNN(\n",
      "  (cnn): Sequential(\n",
      "    (0): Conv2d(3, 64, kernel_size=(9, 9), stride=(2, 2), padding=(3, 3))\n",
      "    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (2): ReLU(inplace=True)\n",
      "    (3): AdaptiveAvgPool2d(output_size=(35, 35))\n",
      "    (4): Conv2d(64, 32, kernel_size=(6, 6), stride=(1, 1))\n",
      "    (5): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (6): Conv2d(32, 16, kernel_size=(3, 3), stride=(1, 1))\n",
      "    (7): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (8): Conv2d(16, 1, kernel_size=(1, 1), stride=(1, 1))\n",
      "  )\n",
      "  (lstm): LSTM(28, 512, num_layers=2, batch_first=True)\n",
      "  (fc): Linear(in_features=512, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "model = CRNN(input_size, hidden_size, num_layers, num_classes).to(device)\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "5763bdd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=gamma) # 学习方式\n",
    "# Train the model\n",
    "total_step = len(train_loader)\n",
    "train_acc, val_acc = [], []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "250fbfd9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:28<00:00, 12.46it/s]\n",
      "100%|██████████| 365/365 [00:22<00:00, 16.37it/s]\n",
      "  0%|          | 2/1097 [00:00<01:20, 13.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 0, train loss:1.6902, train acc:0.4240\n",
      "val acc:0.5139\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:28<00:00, 12.43it/s]\n",
      "100%|██████████| 365/365 [00:19<00:00, 19.11it/s]\n",
      "  0%|          | 2/1097 [00:00<01:15, 14.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 1, train loss:1.0579, train acc:0.6368\n",
      "val acc:0.5107\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:28<00:00, 12.44it/s]\n",
      "100%|██████████| 365/365 [00:21<00:00, 16.68it/s]\n",
      "  0%|          | 1/1097 [00:00<01:50,  9.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 2, train loss:0.8728, train acc:0.6982\n",
      "val acc:0.5528\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:28<00:00, 12.44it/s]\n",
      "100%|██████████| 365/365 [00:21<00:00, 16.99it/s]\n",
      "  0%|          | 2/1097 [00:00<01:15, 14.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 3, train loss:0.7674, train acc:0.7323\n",
      "val acc:0.5191\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:28<00:00, 12.44it/s]\n",
      "100%|██████████| 365/365 [00:19<00:00, 18.66it/s]\n",
      "  0%|          | 2/1097 [00:00<01:16, 14.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 4, train loss:0.6959, train acc:0.7544\n",
      "val acc:0.5065\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:28<00:00, 12.41it/s]\n",
      "100%|██████████| 365/365 [00:21<00:00, 16.67it/s]\n",
      "  0%|          | 2/1097 [00:00<01:34, 11.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 5, train loss:0.6221, train acc:0.7800\n",
      "val acc:0.4920\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:28<00:00, 12.37it/s]\n",
      "100%|██████████| 365/365 [00:18<00:00, 19.22it/s]\n",
      "  0%|          | 2/1097 [00:00<01:13, 14.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 6, train loss:0.5324, train acc:0.8097\n",
      "val acc:0.5231\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:28<00:00, 12.38it/s]\n",
      "100%|██████████| 365/365 [00:21<00:00, 16.65it/s]\n",
      "  0%|          | 2/1097 [00:00<01:13, 14.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 7, train loss:0.4181, train acc:0.8498\n",
      "val acc:0.4993\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:28<00:00, 12.36it/s]\n",
      "100%|██████████| 365/365 [00:21<00:00, 16.94it/s]\n",
      "  0%|          | 2/1097 [00:00<01:19, 13.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 8, train loss:0.3056, train acc:0.8928\n",
      "val acc:0.4735\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:28<00:00, 12.36it/s]\n",
      "100%|██████████| 365/365 [00:19<00:00, 19.12it/s]\n",
      "  0%|          | 2/1097 [00:00<01:13, 14.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 9, train loss:0.2121, train acc:0.9265\n",
      "val acc:0.4967\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:28<00:00, 12.37it/s]\n",
      "100%|██████████| 365/365 [00:21<00:00, 16.74it/s]\n",
      "  0%|          | 2/1097 [00:00<01:19, 13.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:10, train loss:0.1588, train acc:0.9464\n",
      "val acc:0.4828\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:28<00:00, 12.40it/s]\n",
      "100%|██████████| 365/365 [00:21<00:00, 17.01it/s]\n",
      "  0%|          | 2/1097 [00:00<01:17, 14.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:11, train loss:0.1325, train acc:0.9544\n",
      "val acc:0.4709\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:28<00:00, 12.44it/s]\n",
      "100%|██████████| 365/365 [00:21<00:00, 16.75it/s]\n",
      "  0%|          | 2/1097 [00:00<01:15, 14.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:12, train loss:0.1207, train acc:0.9592\n",
      "val acc:0.4778\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:28<00:00, 12.42it/s]\n",
      "100%|██████████| 365/365 [00:21<00:00, 16.98it/s]\n",
      "  0%|          | 2/1097 [00:00<01:22, 13.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:13, train loss:0.1052, train acc:0.9649\n",
      "val acc:0.4770\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:28<00:00, 12.44it/s]\n",
      "100%|██████████| 365/365 [00:22<00:00, 16.07it/s]\n",
      "  0%|          | 2/1097 [00:00<01:22, 13.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:14, train loss:0.1032, train acc:0.9647\n",
      "val acc:0.4622\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:34<00:00, 11.59it/s]\n",
      "100%|██████████| 365/365 [00:24<00:00, 14.92it/s]\n",
      "  0%|          | 2/1097 [00:00<01:37, 11.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:15, train loss:0.0929, train acc:0.9687\n",
      "val acc:0.4731\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:34<00:00, 11.60it/s]\n",
      "100%|██████████| 365/365 [00:21<00:00, 16.79it/s]\n",
      "  0%|          | 2/1097 [00:00<01:24, 13.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:16, train loss:0.0981, train acc:0.9663\n",
      "val acc:0.4844\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:34<00:00, 11.59it/s]\n",
      "100%|██████████| 365/365 [00:25<00:00, 14.32it/s]\n",
      "  0%|          | 1/1097 [00:00<02:48,  6.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:17, train loss:0.0807, train acc:0.9729\n",
      "val acc:0.4753\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:34<00:00, 11.60it/s]\n",
      "100%|██████████| 365/365 [00:23<00:00, 15.64it/s]\n",
      "  0%|          | 2/1097 [00:00<01:24, 12.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:18, train loss:0.0752, train acc:0.9743\n",
      "val acc:0.4935\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███▋      | 410/1097 [00:35<00:59, 11.60it/s]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "100%|██████████| 1097/1097 [01:34<00:00, 11.65it/s]\n",
      "100%|██████████| 365/365 [00:20<00:00, 18.18it/s]\n",
      "  0%|          | 2/1097 [00:00<01:23, 13.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:23, train loss:0.0765, train acc:0.9737\n",
      "val acc:0.4847\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:34<00:00, 11.62it/s]\n",
      "100%|██████████| 365/365 [00:25<00:00, 14.44it/s]\n",
      "  0%|          | 2/1097 [00:00<01:21, 13.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:24, train loss:0.0655, train acc:0.9781\n",
      "val acc:0.4877\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:34<00:00, 11.63it/s]\n",
      "100%|██████████| 365/365 [00:21<00:00, 17.35it/s]\n",
      "  0%|          | 2/1097 [00:00<01:26, 12.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:25, train loss:0.0646, train acc:0.9774\n",
      "val acc:0.4719\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1097/1097 [01:34<00:00, 11.63it/s]\n",
      "100%|██████████| 365/365 [00:23<00:00, 15.77it/s]\n",
      "  0%|          | 2/1097 [00:00<01:25, 12.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH:26, train loss:0.0665, train acc:0.9773\n",
      "val acc:0.4869\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▍     | 484/1097 [00:41<00:52, 11.63it/s]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(num_epochs):\n",
    "    epoch_accuracy = 0\n",
    "    epoch_loss = 0\n",
    "    model.train()\n",
    "    for i, (images, labels) in enumerate(tqdm(train_loader)):\n",
    "        # reshape的维度变成了(batch size, 28, 28)\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        output = model(images)\n",
    "        loss = criterion(output, labels)\n",
    "        # Backward and optimize\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        acc = (output.argmax(dim=1) == labels).float().mean()\n",
    "        epoch_accuracy += acc / len(train_loader)\n",
    "        epoch_loss += loss / len(train_loader)\n",
    "\n",
    "    # val the model\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        epoch_val_accuracy = 0\n",
    "        epoch_val_loss = 0\n",
    "        for images, labels in tqdm(val_loader):\n",
    "            images = images.to(device)\n",
    "            labels = labels.to(device)\n",
    "            outputs = model(images)\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            acc = (predicted == labels).float().mean()\n",
    "            epoch_val_accuracy += acc / len(val_loader)\n",
    "\n",
    "    scheduler.step()\n",
    "    print(f'EPOCH:{epoch:2}, train loss:{epoch_loss:.4f}, train acc:{epoch_accuracy:.4f}')\n",
    "    print(f'val acc:{epoch_val_accuracy:.4f}')\n",
    "\n",
    "    train_acc.append(epoch_accuracy)\n",
    "    val_acc.append(epoch_val_accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6f4cfa4",
   "metadata": {},
   "source": [
    "# KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "84192302",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "e562d6da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.30752467632354824"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier(n_neighbors=1, n_jobs=-1).fit(train_sp, train_label)\n",
    "knn.score(val_sp, val_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cfca56d5",
   "metadata": {},
   "source": [
    "# 1D CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "id": "020e7546",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "#import config\n",
    "\n",
    "\n",
    "class SampleCNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(SampleCNN, self).__init__()\n",
    "\n",
    "        # 59049 x 1\n",
    "        self.conv1 = nn.Sequential(\n",
    "            nn.Conv1d(1, 128, kernel_size=3, stride=3, padding=0),\n",
    "            nn.BatchNorm1d(128),\n",
    "            nn.ReLU())\n",
    "        # 19683 x 128\n",
    "        self.conv2 = nn.Sequential(\n",
    "            nn.Conv1d(128, 128, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm1d(128),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool1d(3, stride=3))\n",
    "        # 6561 x 128\n",
    "        self.conv3 = nn.Sequential(\n",
    "            nn.Conv1d(128, 128, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm1d(128),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool1d(3,stride=3))\n",
    "        # 2187 x 128\n",
    "        self.conv4 = nn.Sequential(\n",
    "            nn.Conv1d(128, 256, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm1d(256),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool1d(3,stride=3))\n",
    "        # 729 x 256\n",
    "        self.conv5 = nn.Sequential(\n",
    "            nn.Conv1d(256, 256, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm1d(256),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool1d(3,stride=3))\n",
    "        # 243 x 256\n",
    "        self.conv6 = nn.Sequential(\n",
    "            nn.Conv1d(256, 256, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm1d(256),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool1d(3,stride=3),\n",
    "            nn.Dropout(0.2))\n",
    "        # 81 x 256\n",
    "        self.conv7 = nn.Sequential(\n",
    "            nn.Conv1d(256, 256, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm1d(256),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool1d(3,stride=3))\n",
    "        # 27 x 256\n",
    "        self.conv8 = nn.Sequential(\n",
    "            nn.Conv1d(256, 256, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm1d(256),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool1d(3,stride=3))\n",
    "        # 9 x 256\n",
    "        self.conv9 = nn.Sequential(\n",
    "            nn.Conv1d(256, 256, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm1d(256),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool1d(3,stride=3))\n",
    "        # 3 x 256\n",
    "        self.conv10 = nn.Sequential(\n",
    "            nn.Conv1d(256, 512, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm1d(512),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool1d(3,stride=3))\n",
    "        # 1 x 512 \n",
    "        self.conv11 = nn.Sequential(\n",
    "            nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n",
    "            nn.BatchNorm1d(512),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(0.2))\n",
    "        # 1 x 512 \n",
    "        self.fc1 = nn.Linear(512, 256)\n",
    "        self.fc2 = nn.Linear(256, 10)\n",
    "        self.activation = nn.Sigmoid()\n",
    "    \n",
    "    def forward(self, x):\n",
    "        # input x : 23 x 59049 x 1\n",
    "        # expected conv1d input : minibatch_size x num_channel x width\n",
    "\n",
    "        # 这里只是将数据增加一个维度\n",
    "        x = x.view(x.shape[0], 1,-1)\n",
    "\n",
    "        out = self.conv1(x)\n",
    "        out = self.conv2(out)\n",
    "        out = self.conv3(out)\n",
    "        out = self.conv4(out)\n",
    "        #out = self.conv5(out)\n",
    "        #print(out.shape)\n",
    "        #out = self.conv6(out)\n",
    "        #print(out.shape)\n",
    "        #out = self.conv7(out)\n",
    "        #print(out.shape)\n",
    "        #assert 0\n",
    "        out = self.conv8(out)\n",
    "        out = self.conv9(out)\n",
    "        out = self.conv10(out)\n",
    "        out = self.conv11(out) \n",
    "        out = out.view(x.shape[0], out.size(1) * out.size(2))\n",
    "        logit = self.fc1(out)\n",
    "        logit = self.fc2(logit)\n",
    "        return logit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "id": "419c8d0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_fft_and_scaler(data, start=5192, end=8192):\n",
    "    data = np.fft.fft(data)\n",
    "    data = np.abs(data)\n",
    "    data = data/np.expand_dims(data.max(axis=1), axis=1)\n",
    "    return data[:,start:end]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "8d7a7440",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sp = get_fft_and_scaler(train)\n",
    "test_sp = get_fft_and_scaler(test)\n",
    "val_sp = get_fft_and_scaler(val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "id": "3c5b31e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SampleCNN().cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "id": "67bac141",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 0.0001\n",
    "gamma = 1\n",
    "step_size = 1\n",
    "EPOCH = 80\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "lf = nn.CrossEntropyLoss()\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=gamma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "id": "b525e263",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "\n",
    "train_tensor = torch.tensor(train_sp).float()\n",
    "y_train_tensor = torch.tensor(train_label).long()\n",
    "val_tensor = torch.tensor(val_sp).float()\n",
    "y_val_tensor = torch.tensor(val_label).long()\n",
    "\n",
    "# 使用Dataloader对数据进行封装\n",
    "train_tensor = TensorDataset(train_tensor, y_train_tensor)\n",
    "val_tensor = TensorDataset(val_tensor, y_val_tensor)\n",
    "\n",
    "\n",
    "train_loader = DataLoader(train_tensor, shuffle=True, batch_size=batch_size, drop_last=True)\n",
    "val_loader = DataLoader(val_tensor, shuffle=False, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "id": "43be7546",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:27<00:00, 20.15it/s]\n",
      "100%|██████████| 183/183 [00:02<00:00, 87.87it/s]\n",
      "  0%|          | 2/548 [00:00<00:28, 19.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0 loss:0.936871 train_acc:0.653136\n",
      "val_acc:0.541084\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:27<00:00, 20.19it/s]\n",
      "100%|██████████| 183/183 [00:02<00:00, 88.42it/s]\n",
      "  0%|          | 2/548 [00:00<00:32, 16.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1 loss:0.706211 train_acc:0.763278\n",
      "val_acc:0.519805\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:27<00:00, 20.11it/s]\n",
      "100%|██████████| 183/183 [00:02<00:00, 84.51it/s]\n",
      "  0%|          | 2/548 [00:00<00:30, 18.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 2 loss:0.699472 train_acc:0.796038\n",
      "val_acc:0.547366\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:27<00:00, 20.02it/s]\n",
      "100%|██████████| 183/183 [00:02<00:00, 86.36it/s]\n",
      "  0%|          | 2/548 [00:00<00:30, 18.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 3 loss:0.637342 train_acc:0.811662\n",
      "val_acc:0.484852\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:27<00:00, 20.11it/s]\n",
      "100%|██████████| 183/183 [00:02<00:00, 86.45it/s]\n",
      "  0%|          | 2/548 [00:00<00:28, 19.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 4 loss:0.424068 train_acc:0.825350\n",
      "val_acc:0.459898\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:27<00:00, 20.29it/s]\n",
      "100%|██████████| 183/183 [00:02<00:00, 88.79it/s]\n",
      "  0%|          | 2/548 [00:00<00:27, 19.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 5 loss:0.496331 train_acc:0.833725\n",
      "val_acc:0.518694\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:26<00:00, 20.30it/s]\n",
      "100%|██████████| 183/183 [00:02<00:00, 88.78it/s]\n",
      "  0%|          | 2/548 [00:00<00:27, 19.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 6 loss:0.483861 train_acc:0.842285\n",
      "val_acc:0.484596\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:27<00:00, 20.19it/s]\n",
      "100%|██████████| 183/183 [00:02<00:00, 88.72it/s]\n",
      "  0%|          | 2/548 [00:00<00:27, 19.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 7 loss:0.533973 train_acc:0.848638\n",
      "val_acc:0.491945\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:26<00:00, 20.38it/s]\n",
      "100%|██████████| 183/183 [00:02<00:00, 88.88it/s]\n",
      "  0%|          | 2/548 [00:00<00:27, 19.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 8 loss:0.431098 train_acc:0.856557\n",
      "val_acc:0.543648\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:26<00:00, 20.38it/s]\n",
      "100%|██████████| 183/183 [00:02<00:00, 89.01it/s]\n",
      "  0%|          | 2/548 [00:00<00:27, 19.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 9 loss:0.362837 train_acc:0.860858\n",
      "val_acc:0.554459\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:26<00:00, 20.40it/s]\n",
      "100%|██████████| 183/183 [00:02<00:00, 89.09it/s]\n",
      "  0%|          | 2/548 [00:00<00:28, 19.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:10 loss:0.389495 train_acc:0.867610\n",
      "val_acc:0.475879\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:26<00:00, 20.40it/s]\n",
      "100%|██████████| 183/183 [00:02<00:00, 89.21it/s]\n",
      "  0%|          | 2/548 [00:00<00:27, 19.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:11 loss:0.456807 train_acc:0.873563\n",
      "val_acc:0.521429\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:27<00:00, 20.19it/s]\n",
      "100%|██████████| 183/183 [00:02<00:00, 86.70it/s]\n",
      "  0%|          | 2/548 [00:00<00:27, 19.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:12 loss:0.320870 train_acc:0.879503\n",
      "val_acc:0.505747\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:27<00:00, 20.17it/s]\n",
      "100%|██████████| 183/183 [00:02<00:00, 86.22it/s]\n",
      "  0%|          | 2/548 [00:00<00:28, 19.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:13 loss:0.451901 train_acc:0.884730\n",
      "val_acc:0.509593\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:27<00:00, 20.17it/s]\n",
      "100%|██████████| 183/183 [00:02<00:00, 86.50it/s]\n",
      "  0%|          | 2/548 [00:00<00:28, 19.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:14 loss:0.393888 train_acc:0.891268\n",
      "val_acc:0.525146\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:27<00:00, 20.15it/s]\n",
      "100%|██████████| 183/183 [00:02<00:00, 86.38it/s]\n",
      "  0%|          | 2/548 [00:00<00:28, 18.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:15 loss:0.411403 train_acc:0.897406\n",
      "val_acc:0.478272\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:27<00:00, 20.13it/s]\n",
      "100%|██████████| 183/183 [00:02<00:00, 86.32it/s]\n",
      "  0%|          | 2/548 [00:00<00:28, 18.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:16 loss:0.348050 train_acc:0.902918\n",
      "val_acc:0.482887\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 548/548 [00:27<00:00, 20.15it/s]\n",
      "100%|██████████| 183/183 [00:02<00:00, 86.52it/s]\n",
      "  0%|          | 2/548 [00:00<00:28, 19.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:17 loss:0.209811 train_acc:0.907775\n",
      "val_acc:0.492843\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|█████▉    | 328/548 [00:16<00:10, 20.16it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-236-2284e356782a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m         \u001b[0mtrain_total_acc\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeature\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meq\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/optim/lr_scheduler.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     63\u001b[0m                 \u001b[0minstance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_step_count\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m                 \u001b[0mwrapped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minstance\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mwrapped\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m             \u001b[0;31m# Note that the returned function here is no longer a bound method,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/optim/optimizer.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     87\u001b[0m                 \u001b[0mprofile_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"Optimizer.step#{}.step\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecord_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprofile_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m                     \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     90\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    106\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    107\u001b[0m             \u001b[0mbeta1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbeta2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'betas'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 108\u001b[0;31m             F.adam(params_with_grad,\n\u001b[0m\u001b[1;32m    109\u001b[0m                    \u001b[0mgrads\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    110\u001b[0m                    \u001b[0mexp_avgs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/asr/lib/python3.8/site-packages/torch/optim/_functional.py\u001b[0m in \u001b[0;36madam\u001b[0;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, amsgrad, beta1, beta2, lr, weight_decay, eps)\u001b[0m\n\u001b[1;32m     90\u001b[0m             \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mmax_exp_avg_sqs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias_correction2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m             \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias_correction2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m         \u001b[0mstep_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlr\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mbias_correction1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "train_best = 1000\n",
    "for epoch in range(EPOCH):\n",
    "    model.train()\n",
    "    train_total_acc = 0\n",
    "    \n",
    "    for feature, label in tqdm(train_loader):\n",
    "        feature = feature.cuda()\n",
    "        label = label.cuda()\n",
    "        optimizer.zero_grad()\n",
    "        preds = model(feature)\n",
    "        loss = lf(preds, label)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        train_total_acc += model(feature).argmax(dim=1).eq(label).sum().item()\n",
    "        \n",
    "        feature.cpu()\n",
    "        label.cpu()\n",
    "    \n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        val_total_acc = 0\n",
    "        for feature, label in tqdm(val_loader):\n",
    "            feature = feature.cuda()\n",
    "            label = label.cuda()\n",
    "            val_total_acc += model(feature).argmax(dim=1).eq(label).sum().item()\n",
    "            \n",
    "            feature.cpu()\n",
    "            label.cpu()\n",
    "    \n",
    "    scheduler.step()\n",
    "    print(f'epoch:{epoch:2} loss:{loss:4f} train_acc:{train_total_acc/len(train_label):4f}')\n",
    "    print('val_acc:{:4f}'.format(val_total_acc/len(val_label)))\n",
    "    \n",
    "    # 保存最佳的训练模型\n",
    "    if loss <= train_best:\n",
    "        train_best = loss\n",
    "        torch.save(model.state_dict(), './model.point')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "270d0b79",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a8735c1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "165px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
